def test_cce_one_hot(): y_a = K.variable(np.random.randint(0, 7, (5, 6))) y_b = K.variable(np.random.random((5, 6, 7))) objective_output = sparse_categorical_crossentropy(y_a, y_b) assert K.eval(objective_output).shape == (5, 6) y_a = K.variable(np.random.randint(0, 7, (6,))) y_b = K.variable(np.random.random((6, 7))) assert K.eval(sparse_categorical_crossentropy(y_a, y_b)).shape == (6,)
def transformer_loss(target_waveform, pred_waveform, target_genres, pred_genres): waveform_loss = sparse_categorical_crossentropy(target_waveform, pred_waveform, from_logits=True) genre_loss = sparse_categorical_crossentropy(target_genres, pred_genres, from_logits=True) return tf.reduce_sum(waveform_loss, axis=-1) - 0.01 * 44100 * genre_loss
def call(self, inputs): # update key extractor weights k_weights = [self.m*w_k + (1-self.m)*w_q for w_k, w_q \ in zip(self.k_enc.get_weights(), self.q_enc.get_weights())] self.k_enc.set_weights(k_weights) # get two versions of same batch data x_q, x_k = self.rand_aug(inputs) # save the key and query # sample_q = tf.io.encode_jpeg(tf.cast(x_q[0]*255, tf.uint8)) # ts = ''.join(map(str, list(time.localtime()))) # tf.io.write_file(f'{ts}_sample_q.jpeg', sample_q) # sample_k = tf.io.encode_jpeg(tf.cast(x_k[0]*255, tf.uint8)) # tf.io.write_file(f'{ts}_sample_k.jpeg', sample_k) # forward q = self.g(self.q_enc(x_q)) q = tf.reshape(q, (tf.shape(q)[0], 1, -1)) k = self.g(self.k_enc(x_k)) k = tf.reshape(k, (tf.shape(k)[0], -1, 1)) l_pos = tf.squeeze(tf.matmul(q, k), axis=-1) l_neg = tf.matmul(tf.squeeze(q), tf.transpose(self.queue)) # logits = softmax(tf.concat([l_pos, l_neg], axis=1)) logits = tf.concat([l_pos, l_neg], axis=1) self.queue_them(tf.squeeze(k)) ###### keras-fashion version ###### # return logits ###### gradient-tape version ###### labels = tf.zeros(tf.shape(inputs)[0]) loss = K.mean(sparse_categorical_crossentropy(labels, logits, from_logits=True)) l2 = tf.reduce_mean(tf.math.l2_normalize(q)) # print(K.max(logits, axis=1).numpy()) hits = tf.equal(tf.argmax(logits, axis=1), tf.cast(labels, 'int64')) acc = tf.reduce_mean(tf.cast(hits, 'float64')) return loss + 0.1 * l2, acc
def step(self, x_true, y_true): with tf.GradientTape() as tape: pred = self.model(x_true) loss = sparse_categorical_crossentropy(y_true, pred) grads = tape.gradient(loss, self.model.trainable_variables) self.opt.apply_gradients(zip(grads, self.model.trainable_variables))
def custom_loss(y_true, y_pred): loss = sparse_categorical_crossentropy(y_true, y_pred) if extra_losses is not None: for fn in extra_losses: loss += fn(model) return loss
def yolo_loss(y_true, y_pred): # 1. transform all pred outputs # y_pred: (batch_size, grid, grid, anchors, (x, y, w, h, obj, ...cls)) pred_box, pred_obj, pred_class, pred_xywh = yolo_boxes( y_pred, anchors, classes) pred_xy = pred_xywh[..., 0:2] pred_wh = pred_xywh[..., 2:4] # 2. transform all true outputs # y_true: (batch_size, grid, grid, anchors, (x1, y1, x2, y2, obj, cls)) true_box, true_obj, true_class_idx = tf.split(y_true, (4, 1, 1), axis=-1) true_xy = (true_box[..., 0:2] + true_box[..., 2:4]) / 2 true_wh = true_box[..., 2:4] - true_box[..., 0:2] # give higher weights to small boxes box_loss_scale = 2 - true_wh[..., 0] * true_wh[..., 1] # 3. inverting the pred box equations grid_size = tf.shape(y_true)[1] grid = tf.meshgrid(tf.range(grid_size), tf.range(grid_size)) grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2) true_xy = true_xy * tf.cast(grid_size, tf.float32) - \ tf.cast(grid, tf.float32) true_wh = tf.math.log(true_wh / anchors) true_wh = tf.where(tf.math.is_inf(true_wh), tf.zeros_like(true_wh), true_wh) # 4. calculate all masks obj_mask = tf.squeeze(true_obj, -1) # ignore false positive when iou is over threshold best_iou = tf.map_fn( lambda x: tf.reduce_max(broadcast_iou( x[0], tf.boolean_mask(x[1], tf.cast(x[2], tf.bool))), axis=-1), (pred_box, true_box, obj_mask), tf.float32) ignore_mask = tf.cast(best_iou < ignore_thresh, tf.float32) # 5. calculate all losses xy_loss = obj_mask * box_loss_scale * \ tf.reduce_sum(tf.square(true_xy - pred_xy), axis=-1) wh_loss = obj_mask * box_loss_scale * \ tf.reduce_sum(tf.square(true_wh - pred_wh), axis=-1) obj_loss = binary_crossentropy(true_obj, pred_obj) obj_loss = obj_mask * obj_loss + \ (1 - obj_mask) * ignore_mask * obj_loss # TODO: use binary_crossentropy instead if FLAGS.num_classes == 1: class_loss = obj_mask * binary_crossentropy( true_class_idx, pred_class, from_logits=True) else: class_loss = obj_mask * sparse_categorical_crossentropy( true_class_idx, pred_class, from_logits=True) # 6. sum over (batch, gridx, gridy, anchors) => (batch, 1) xy_loss = tf.reduce_sum(xy_loss, axis=(1, 2, 3)) wh_loss = tf.reduce_sum(wh_loss, axis=(1, 2, 3)) obj_loss = tf.reduce_sum(obj_loss, axis=(1, 2, 3)) class_loss = tf.reduce_sum(class_loss, axis=(1, 2, 3)) return xy_loss + wh_loss + obj_loss + class_loss
def yolo_loss(y_true, y_pred): """ Code from: https://github.com/zzh8829/yolov3-tf2/blob/master/yolov3_tf2/dataset.py """ # 1. transform all pred outputs # y_pred: (batch_size, grid, grid, anchors, (x, y, w, h, obj, ...cls)) pred_box, pred_obj, pred_class, pred_xywh = self.extract_from_predictions( y_pred, scale_index) pred_xy = pred_xywh[..., 0:2] pred_wh = pred_xywh[..., 2:4] # 2. transform all true outputs # y_true: (batch_size, grid, grid, anchors, (x1, y1, x2, y2, obj, cls)) true_box, true_obj, true_class_idx = tf.split(y_true, (4, 1, 1), axis=-1) true_xy = (true_box[..., 0:2] + true_box[..., 2:4]) / 2 true_wh = true_box[..., 2:4] - true_box[..., 0:2] # give higher weights to small boxes box_loss_scale = 2 - true_wh[..., 0] * true_wh[..., 1] # 3. inverting the pred box equations grid_size = tf.shape(y_true)[1] grid = tf.meshgrid(tf.range(grid_size), tf.range(grid_size)) grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2) true_xy = true_xy * tf.cast(grid_size, tf.float32) - tf.cast( grid, tf.float32) true_wh = tf.math.log(true_wh / self.anchors[self.masks[scale_index]]) true_wh = tf.where(tf.math.is_inf(true_wh), tf.zeros_like(true_wh), true_wh) # 4. calculate all masks obj_mask = tf.squeeze(true_obj, -1) # ignore false positive when iou is over threshold true_box_flat = tf.boolean_mask(true_box, tf.cast(obj_mask, tf.bool)) best_iou = tf.reduce_max(helpers.broadcast_iou( pred_box, true_box_flat), axis=-1) ignore_mask = tf.cast(best_iou < self.iou_threshold, tf.float32) # 5. calculate all losses xy_loss = obj_mask * box_loss_scale * tf.reduce_sum( tf.square(true_xy - pred_xy), axis=-1) wh_loss = obj_mask * box_loss_scale * tf.reduce_sum( tf.square(true_wh - pred_wh), axis=-1) obj_loss = binary_crossentropy(true_obj, pred_obj) obj_loss = obj_mask * obj_loss + ( 1 - obj_mask) * ignore_mask * obj_loss class_loss = obj_mask * sparse_categorical_crossentropy( true_class_idx, pred_class) # 6. sum over (batch, gridx, gridy, anchors) => (batch, 1) xy_loss = tf.reduce_sum(xy_loss, axis=(1, 2, 3)) wh_loss = tf.reduce_sum(wh_loss, axis=(1, 2, 3)) obj_loss = tf.reduce_sum(obj_loss, axis=(1, 2, 3)) class_loss = tf.reduce_sum(class_loss, axis=(1, 2, 3)) return xy_loss + wh_loss + obj_loss + class_loss
def _build_net(self): input = layers.Input(shape=(self.n_features,)) x = layers.Dense(10, activation="tanh")(input) self.all_act = layers.Dense(self.n_actions)(x) loss = losses.sparse_categorical_crossentropy() rmsprop = optimizers.RMSprop(lr=self.learning_rate) self.model = models.Model(input, self.all_act) self.model.compile(loss=loss, optimizer=rmsprop, metrics=['accuracy'])
def compute_loss(y_true, y_pred): M = tf.convert_to_tensor(y_true[:, :, :, :, 0]) W = tf.convert_to_tensor(y_true[:, :, :, :, 1]) y_pred = tf.convert_to_tensor(y_pred) y1 = sparse_categorical_crossentropy(M, y_pred) y2 = tf.math.multiply(W, y1) y1 = tf.reduce_mean(y1) y2 = tf.reduce_mean(y2) y = y1 + alpha * y2 return y
def compute_loss(y_true, y_pred): """ 计算loss :param y_true: 模型 :param y_pred: :return: """ mse = losses.sparse_categorical_crossentropy(y_true, y_pred, from_logits=True) return mse
def yolo_loss(y_true, y_pred): # y_pred: (batch_size, grid, grid, anchors, (x, y, w, h, obj, ...cls)) pred_box, pred_obj, pred_class, pred_xywh = yoloBoxes( y_pred, anchors, classes) pred_xy = pred_xywh[..., 0:2] pred_wh = pred_xywh[..., 2:4] # PASCAL VOC -> COCO # xmin, ymin, xmax, ymax -> x, y, w, h # y_true: (batch_size, grid, grid, anchors, (x1, y1, x2, y2, obj, cls)) true_box, true_obj, true_class_idx = tf.split(y_true, (4, 1, 1), axis=-1) true_xy = (true_box[..., 0:2] + true_box[..., 2:4]) / 2 true_wh = true_box[..., 2:4] - true_box[..., 0:2] # give higher weights to small boxes # true_wh[..., 0] * true_wh[..., 1] : box area box_loss_scale = 2 - true_wh[..., 0] * true_wh[..., 1] grid_size = tf.shape(y_true)[1] grid = tf.meshgrid(tf.range(grid_size), tf.range(grid_size)) grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2) true_xy = true_xy * tf.cast(grid_size, tf.float32) - tf.cast( grid, tf.float32) true_wh = tf.math.log(true_wh / anchors) true_wh = tf.where(tf.math.is_inf(true_wh), tf.zeros_like(true_wh), true_wh) obj_mask = tf.squeeze(true_obj, -1) best_iou = tf.map_fn( lambda x: tf.reduce_max(broadcast_iou( x[0], tf.boolean_mask(x[1], tf.cast(x[2], tf.bool))), axis=-1), (pred_box, true_box, obj_mask), tf.float32) ignore_mask = tf.cast(best_iou < ignore_thresh, tf.float32) xy_loss = obj_mask * box_loss_scale * tf.reduce_sum( tf.square(true_xy - pred_xy), axis=-1) wh_loss = obj_mask * box_loss_scale * tf.reduce_sum( tf.square(true_wh - pred_wh), axis=-1) obj_loss = binary_crossentropy(true_obj, pred_obj) obj_loss = obj_mask * obj_loss + (1 - obj_mask) * ignore_mask * obj_loss class_loss = obj_mask * sparse_categorical_crossentropy( true_class_idx, pred_class) xy_loss = tf.reduce_sum(xy_loss, axis=(1, 2, 3)) wh_loss = tf.reduce_sum(wh_loss, axis=(1, 2, 3)) obj_loss = tf.reduce_sum(obj_loss, axis=(1, 2, 3)) class_loss = tf.reduce_sum(class_loss, axis=(1, 2, 3)) return xy_loss + wh_loss + obj_loss + class_loss
def custom_loss(y_true, y_pred): # y_true[i] = (row, col, height, width, # p_class1, p_class2, p_class3, # p(object_appeared|img)) # the bounding box loss: bce_1 = binary_crossentropy(y_true[:, :4], y_pred[:, :4]) # the object class prediction loss: scce = sparse_categorical_crossentropy(y_true[:, 4], y_pred[:, 4:-1]) # the binary prediction (about an object being present in an image) loss: bce_2 = binary_crossentropy(y_true[:, -1], y_pred[:, -1]) return ALPHA * y_true[:, -1] * bce_1 + BETA * y_true[:, -1] * scce + GAMMA * bce_2
def do_test_forward(self, sequence): with self.device: self.test_metric.reset_states() for inputs, labels in sequence: x, adj, index, _ = inputs logit = self.propagation(x, adj, training=False) output = tf.gather(logit, index) output = softmax(output) loss = tf.reduce_mean(sparse_categorical_crossentropy(labels, output)) self.test_metric.update_state(labels, output) return loss, self.test_metric.result()
def yolo_loss(y_true, y_pred): # 1. transform all pred outputs # y_pred: (batch_size, grid, grid, anchors, (x, y, w, h, obj, ...cls)) pred_box, pred_obj, pred_class = yolo_boxes(y_pred, anchors, classes, calc_loss=True) pred_xy = (pred_box[..., 0:2] + pred_box[..., 2:4]) / 2 pred_wh = pred_box[..., 2:4] - pred_box[..., 0:2] # 2. transform all true outputs # y_true: (batch_size, grid, grid, anchors, (x1, y1, x2, y2, obj, cls)) true_box, true_obj, true_class_idx = tf.split(y_true, (4, 1, 1), axis=-1) true_xy = (true_box[..., 0:2] + true_box[..., 2:4]) / 2 true_wh = true_box[..., 2:4] - true_box[..., 0:2] # give higher weights to small boxes box_loss_scale = 2 - true_wh[..., 0] * true_wh[..., 1] # 4. calculate all masks obj_mask = tf.squeeze(true_obj, -1) # ignore false positive when iou is over threshold best_iou = tf.map_fn( lambda x: tf.reduce_max(broadcast_iou( x[0], tf.boolean_mask(x[1], tf.cast(x[2], tf.bool))), axis=-1), (pred_box, true_box, obj_mask), tf.float32) ignore_mask = tf.cast(best_iou < ignore_thresh, tf.float32) # 5. calculate all losses xy_loss = obj_mask * box_loss_scale * \ tf.reduce_sum(tf.square(true_xy - pred_xy), axis=-1) wh_loss = obj_mask * box_loss_scale * \ tf.reduce_sum(tf.square(true_wh - pred_wh), axis=-1) obj_loss = binary_crossentropy(true_obj, pred_obj, from_logits=True) obj_loss = obj_mask * obj_loss + \ (1 - obj_mask) * ignore_mask * obj_loss # TODO: use binary_crossentropy instead class_loss = obj_mask * sparse_categorical_crossentropy( true_class_idx, pred_class, from_logits=True) # 6. sum over (batch, gridx, gridy, anchors) => (batch, 1) xy_loss = tf.reduce_sum(xy_loss, axis=(1, 2, 3)) wh_loss = tf.reduce_sum(wh_loss, axis=(1, 2, 3)) obj_loss = tf.reduce_sum(obj_loss, axis=(1, 2, 3)) class_loss = tf.reduce_sum(class_loss, axis=(1, 2, 3)) return xy_loss + wh_loss + obj_loss + class_loss
def call(self, inputs: list[tf.Tensor], *args, **kwargs) -> tf.Tensor: # (batch_size, label_seq_len) # (batch_size, label_seq_len, vocab_size) # (batch_size, label_seq_len) token_ids, logits, mask = inputs loss = sparse_categorical_crossentropy(token_ids, logits, from_logits=True) mask = tf.cast(mask, loss.dtype) self.add_loss(tf.reduce_sum(loss * mask) / tf.reduce_sum(mask)) predicted_token_ids = tf.math.argmax(logits, axis=-1) is_equal = tf.cast(tf.equal(token_ids, predicted_token_ids), loss.dtype) lm_accuracy = tf.reduce_sum(is_equal * mask) / tf.reduce_sum(mask) self.add_metric(lm_accuracy, name="lm_accuracy") boolean_mask = tf.cast(mask, tf.bool) return tf.ragged.boolean_mask(predicted_token_ids, boolean_mask)
def yoloLoss(y_true, y_pred): pred_box, pred_obj, pred_class, pred_xywh = yolo_boxes( y_pred, anchors, class_num) pred_xy = pred_xywh[..., 0:2] # 取出偏移量 pred_wh = pred_xywh[..., 2:4] true_box, true_obj, true_class_idx = tf.split(y_true, (4, 1, 1), axis=-1) true_xy = (true_box[..., 0:2] + true_box[..., 2:4]) / 2 true_wh = true_box[..., 2:4] - true_box[..., 0:2] box_loss_scale = 2 - true_wh[..., 0] * true_wh[..., 1] grid_size = tf.shape(y_true)[1] grid = tf.meshgrid(tf.range(grid_size), tf.range(grid_size)) grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2) true_xy = true_xy * tf.cast(grid_size, tf.float32) - \ tf.cast(grid, tf.float32) true_wh = tf.math.log(true_wh / anchors) true_wh = tf.where(tf.math.is_inf(true_wh), tf.zeros_like(true_wh), true_wh) # 将真实坐标转变为偏移量用于计算loss obj_mask = tf.squeeze(true_obj, -1) # ignore false positive when iou is over threshold best_iou = tf.map_fn( lambda x: tf.reduce_max(broadcast_iou( x[0], tf.boolean_mask(x[1], tf.cast(x[2], tf.bool))), axis=-1), (pred_box, true_box, obj_mask), tf.float32) ignore_mask = tf.cast(best_iou < ignore_thresh, tf.float32) xy_loss = obj_mask * box_loss_scale * \ tf.reduce_sum(tf.square(true_xy - pred_xy), axis=-1) wh_loss = obj_mask * box_loss_scale * \ tf.reduce_sum(tf.square(true_wh - pred_wh), axis=-1) obj_loss = binary_crossentropy(true_obj, pred_obj) obj_loss = obj_mask * obj_loss + \ (1 - obj_mask) * ignore_mask * obj_loss # TODO: use binary_crossentropy instead class_loss = obj_mask * sparse_categorical_crossentropy( true_class_idx, pred_class) xy_loss = tf.reduce_sum(xy_loss, axis=(1, 2, 3)) wh_loss = tf.reduce_sum(wh_loss, axis=(1, 2, 3)) obj_loss = tf.reduce_sum(obj_loss, axis=(1, 2, 3)) class_loss = tf.reduce_sum(class_loss, axis=(1, 2, 3)) return xy_loss + wh_loss + obj_loss + class_loss
def yolo_loss(y_true, y_pred): pred_box, pred_obj, pred_class, pred_xywh = yolo_boxes( y_pred, anchors, classes) pred_xy = pred_xywh[..., 0:2] pred_wh = pred_xywh[..., 2:4] true_box, true_obj, true_class_idx = tf.split( y_true, (4, 1, 1), axis=-1) true_xy = (true_box[..., 0:2] + true_box[..., 2:4]) / 2 true_wh = true_box[..., 2:4] - true_box[..., 0:2] box_loss_scale = 2 - true_wh[..., 0] * true_wh[..., 1] grid_size = tf.shape(y_true)[1] grid = tf.meshgrid(tf.range(grid_size), tf.range(grid_size)) grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2) true_xy = true_xy * tf.cast(grid_size, tf.float32) - \ tf.cast(grid, tf.float32) true_wh = tf.math.log(true_wh / anchors) true_wh = tf.where(tf.math.is_inf(true_wh), tf.zeros_like(true_wh), true_wh) obj_mask = tf.squeeze(true_obj, -1) # ignore when Intersection Over Union is over threshold true_box_flat = tf.boolean_mask(true_box, tf.cast(obj_mask, tf.bool)) best_iou = tf.reduce_max(intersectionOverUnion( pred_box, true_box_flat), axis=-1) ignore_mask = tf.cast(best_iou < ignore_thresh, tf.float32) xy_loss = obj_mask * box_loss_scale * \ tf.reduce_sum(tf.square(true_xy - pred_xy), axis=-1) wh_loss = obj_mask * box_loss_scale * \ tf.reduce_sum(tf.square(true_wh - pred_wh), axis=-1) obj_loss = binary_crossentropy(true_obj, pred_obj) obj_loss = obj_mask * obj_loss + \ (1 - obj_mask) * ignore_mask * obj_loss class_loss = obj_mask * sparse_categorical_crossentropy( true_class_idx, pred_class) xy_loss = tf.reduce_sum(xy_loss, axis=(1, 2, 3)) wh_loss = tf.reduce_sum(wh_loss, axis=(1, 2, 3)) obj_loss = tf.reduce_sum(obj_loss, axis=(1, 2, 3)) class_loss = tf.reduce_sum(class_loss, axis=(1, 2, 3)) return xy_loss + wh_loss + obj_loss + class_loss
def mask_sparse_cross_entropy(y_true=None, y_pred=None, mask=0): # y_true: [None, steps] # y_pred: [None, steps, num_classes+1], +1 for pad to mask # loss: [None, steps] loss = sparse_categorical_crossentropy(y_true, y_pred) # masks: [None, steps] masks = tf.cast(tf.not_equal(y_true, mask), tf.float32) # masked-loss: [None, steps] loss = tf.multiply(loss, masks) # reduce_mean: shape=() loss = tf.cast(tf.reduce_mean(loss), tf.float32) return loss
def yolo_loss(y_true, y_pred): p_bboxes, p_objectness, p_class, p_original = make_bboxs(y_pred, anchors, classes) p_xy, p_wh = tf.split(p_original, (2, 2), axis=-1) t_xy = (y_true[..., 0:2] + y_true[..., 2:4]) / 2 # Center of each coord t_wh = y_true[..., 2:4] - y_true[..., 0:2] # Width and height for each # Give higher weights to small boxes l_tune = 2 - t_wh[..., 0] * t_wh[..., 1] # Inverting the pred box equations grid_size = tf.shape(y_true)[1] grid = tf.meshgrid(tf.range(grid_size), tf.range(grid_size)) grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2) t_xy = t_xy * tf.cast(grid_size, tf.float32) - tf.cast(grid, tf.float32) t_wh = tf.math.log(t_wh / anchors) t_wh = tf.where(tf.math.is_inf(t_wh), tf.zeros_like(t_wh), t_wh) # Calculate masks obj_mask = tf.squeeze(y_true[..., 4:5], -1) gather_mask = tf.where(tf.not_equal(obj_mask, 0.)) buffer = [0] * 7 for i, el in enumerate([t_wh, p_wh, t_xy, p_xy, y_true[..., 5:6], p_class, l_tune]): buffer[i] = tf.gather_nd(el, gather_mask) t_wh, p_wh, t_xy, p_xy, t_class, p_class, l_tune = buffer # Calculate best IOU between predictions and true objects best_iou = bbox_iou(p_bboxes, tf.boolean_mask(y_true[..., :4], tf.cast(obj_mask, tf.bool))) obj_loss = losses.binary_crossentropy(y_true[..., 4:5], p_objectness) obj_loss = obj_mask * obj_loss + (1 - obj_mask) * obj_loss * tf.cast(best_iou < iou_thresh, tf.float32) # Losses wh_loss = l_tune * tf.reduce_sum(tf.square(t_wh - p_wh), axis=-1) xy_loss = l_tune * tf.reduce_sum(tf.square(t_xy - p_xy), axis=-1) class_loss = losses.sparse_categorical_crossentropy(t_class, p_class) * 20 total_loss = 0 for loss in [xy_loss, wh_loss, obj_loss, class_loss]: total_loss += tf.reduce_sum(loss) return total_loss
def yolo_loss(y_true, y_pred): pred_box, pred_obj, pred_class, pred_xywh = get_boxes( y_pred, anchors, classes) pred_xy = pred_xywh[..., 0:2] pred_wh = pred_xywh[..., 2:4] true_box, true_obj, true_class_idx = tf.split(y_true, (4, 1, 1), axis=-1) true_xy = (true_box[..., 0:2] + true_box[..., 2:4]) / 2 true_wh = true_box[..., 2:4] - true_box[..., 0:2] box_loss_scale = 2 - true_wh[..., 0] * true_wh[..., 1] grid_size = tf.shape(y_true)[1] grid = tf.meshgrid(tf.range(grid_size), tf.range(grid_size)) grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2) true_xy = true_xy * tf.cast(grid_size, tf.float32) - tf.cast( grid, tf.float32) true_wh = tf.math.log(true_wh / anchors) true_wh = tf.where(tf.math.is_inf(true_wh), tf.zeros_like(true_wh), true_wh) obj_mask = tf.squeeze(true_obj, -1) best_iou = tf.map_fn( lambda x: tf.reduce_max( broadcast_iou(x[0], tf.boolean_mask(x[1], tf.cast(x[2], tf.bool))), axis=-1, ), (pred_box, true_box, obj_mask), fn_output_signature=tf.float32, ) ignore_mask = tf.cast(best_iou < ignore_thresh, tf.float32) xy_loss = (obj_mask * box_loss_scale * tf.reduce_sum(tf.square(true_xy - pred_xy), axis=-1)) wh_loss = (obj_mask * box_loss_scale * tf.reduce_sum(tf.square(true_wh - pred_wh), axis=-1)) obj_loss = binary_crossentropy(true_obj, pred_obj) obj_loss = obj_mask * obj_loss + (1 - obj_mask) * ignore_mask * obj_loss class_loss = obj_mask * sparse_categorical_crossentropy( true_class_idx, pred_class) xy_loss = tf.reduce_sum(xy_loss, axis=(1, 2, 3)) wh_loss = tf.reduce_sum(wh_loss, axis=(1, 2, 3)) obj_loss = tf.reduce_sum(obj_loss, axis=(1, 2, 3)) class_loss = tf.reduce_sum(class_loss, axis=(1, 2, 3)) return xy_loss + wh_loss + obj_loss + class_loss
def custom_loss(y_true, y_pred): y_pred = tf.reshape(y_pred, (-1, n_objects, 5 + n_classes)) pred_obj, pred_xywh, pred_class = tf.split(y_pred, [1, 4, n_classes], axis=-1) y_true = tf.reshape(y_true, (-1, n_objects, 6)) true_obj, true_xywh, true_class = tf.split(y_true, [1, 4, 1], axis=-1) # Cross entropy loss for score obj_loss = binary_crossentropy(true_obj, pred_obj) obj_loss = tf.reduce_sum(obj_loss, axis=1) # MSE loss for the BB bb_loss = mean_squared_error(true_xywh, pred_xywh) bb_loss = tf.reduce_sum(bb_loss, axis=1) # We have to convert the class into a vector class_loss = sparse_categorical_crossentropy(true_class, pred_class) class_loss = tf.reduce_sum(class_loss, axis=1) return obj_loss + bb_loss + class_loss
def crf_loss(y_true, y_pred): """General CRF loss function depending on the learning mode. Args: y_true: tensor with true targets. y_pred: tensor with predicted targets. Returns: If the CRF layer is being trained in the join mode, returns the negative log-likelihood. Otherwise returns the categorical crossentropy implemented by the underlying Keras backend. """ crf, idx = y_pred._keras_history[:2] if crf.learn_mode == 'join': return crf_nll(y_true, y_pred) else: if crf.sparse_target: return sparse_categorical_crossentropy(y_true, y_pred) else: return categorical_crossentropy(y_true, y_pred)
def crf_loss(y_true, y_pred): """General CRF loss function depending on the learning mode. # Arguments y_true: tensor with true targets. y_pred: tensor with predicted targets. # Returns If the CRF layer is being trained in the join mode, returns the negative log-likelihood. Otherwise returns the categorical crossentropy implemented by the underlying Keras backend. # About GitHub If you open an issue or a pull request about CRF, please add `cc @lzfelix` to notify Luiz Felix. """ crf, idx = y_pred._keras_history[:2] if crf.learn_mode == 'join': return crf_nll(y_true, y_pred) else: if crf.sparse_target: return sparse_categorical_crossentropy(y_true, y_pred) else: return categorical_crossentropy(y_true, y_pred)
def step(X, y): # get the weights weights = phase2Kernel.forward() conv_bias = model.layers[0].get_weights()[1] model.layers[0].set_weights([weights, conv_bias]) # keep track of our gradients with tf.GradientTape() as tape: # make a prediction using the model and then calculate the # loss pred = model(X) print("Pred shape:", pred.shape) loss = sparse_categorical_crossentropy(y, pred) accuracy = accuracy_score(y, np.argmax(pred, axis=1)) print("loss", tf.math.reduce_mean(loss)) print("accuracy:", accuracy) grads = tape.gradient(loss, model.trainable_variables) print("gradients shapes: ", grads[0].shape, grads[1].shape) final_grad = phase2Kernel.backward(grads[0]) phase2Kernel.phi -= PhaseLR * cp.asnumpy(final_grad) opt.apply_gradients(zip(grads, model.trainable_variables))
def do_train_forward(self, sequence): with self.device: self.train_metric.reset_states() for inputs, labels in sequence: x, adj, index, adv_mask = inputs with tf.GradientTape() as tape: logit = self.propagation(x, adj) output = tf.gather(logit, index) output = softmax(output) loss = tf.reduce_mean(sparse_categorical_crossentropy(labels, output)) entropy_loss = entropy_y_x(logit) vat_loss = self.virtual_adversarial_loss(x, adj, logit=logit, adv_mask=adv_mask) loss += self.p1 * vat_loss + self.p2 * entropy_loss self.train_metric.update_state(labels, output) trainable_variables = self.model.trainable_variables gradients = tape.gradient(loss, trainable_variables) self.optimizer.apply_gradients(zip(gradients, trainable_variables)) return loss, self.train_metric.result()
def yolo_loss(detector_mask, matching_gt_boxes, matching_classes_oh, gt_boxes_grid, y_pred): # detector_mask: [b,16,16,5,1] # matching_gt_boxes: [b,16,16,5,5] x-y-w-h-l # matching_classes_oh: [b,16,16,5,2] l1-l2 # gt_boxes_grid: [b,40,5] x-y-wh-l # y_pred: [b,16,16,5,7] x-y-w-h-conf-l0-l1 IMGSZ = cfg.TRAIN.IMGSZ GRIDSZ = cfg.TRAIN.GRIDSZ ANCHORS = cfg.TRAIN.ANCHORS anchors = np.array(ANCHORS).reshape(5, 2) # create starting position for each grid anchors # [16,16] x_grid = tf.tile(tf.range(GRIDSZ), [GRIDSZ]) # [1,16,16,1,1] # [b,16,16,5,2] x_grid = tf.reshape(x_grid, (1, GRIDSZ, GRIDSZ, 1, 1)) x_grid = tf.cast(x_grid, tf.float32) # [b,16_1,16_2,1,1]=>[b,16_2,16_1,1,1] y_grid = tf.transpose(x_grid, (0, 2, 1, 3, 4)) xy_grid = tf.concat([x_grid, y_grid], axis=-1) # [1,16,16,1,2]=> [b,16,16,5,2] xy_grid = tf.tile(xy_grid, [y_pred.shape[0], 1, 1, 5, 1]) # [b,16,16,5,7] x-y-w-h-conf-l1-l2 pred_xy = tf.sigmoid(y_pred[..., 0:2]) pred_xy = pred_xy + xy_grid # [b,16,16,5,2] pred_wh = tf.exp(y_pred[..., 2:4]) # [b,16,16,5,2] * [5,2] => [b,16,16,5,2] pred_wh = pred_wh * anchors n_detector_mask = tf.reduce_sum(tf.cast(detector_mask > 0., tf.float32)) # [b,16,16,5,1] * [b,16,16,5,2] # xy_loss = detector_mask * tf.square(matching_gt_boxes[..., :2] - pred_xy) / (n_detector_mask + 1e-6) xy_loss = tf.reduce_sum(xy_loss) wh_loss = detector_mask * tf.square(tf.sqrt(matching_gt_boxes[..., 2:4]) - \ tf.sqrt(pred_wh)) / (n_detector_mask + 1e-6) wh_loss = tf.reduce_sum(wh_loss) # 4.1 coordinate loss coord_loss = xy_loss + wh_loss # 4.2 class loss # [b,16,16,5,2] pred_box_class = y_pred[..., 5:] # [b,16,16,5] true_box_class = tf.argmax(matching_classes_oh, -1) # [b,16,16,5] vs [b,16,16,5,2] class_loss = losses.sparse_categorical_crossentropy( \ true_box_class, pred_box_class, from_logits=True) # [b,16,16,5] => [b,16,16,5,1]* [b,16,16,5,1] class_loss = tf.expand_dims(class_loss, -1) * detector_mask class_loss = tf.reduce_sum(class_loss) / (n_detector_mask + 1e-6) # 4.3 object loss # nonobject_mask # iou done! # [b,16,16,5] x1, y1, w1, h1 = matching_gt_boxes[..., 0], matching_gt_boxes[..., 1], \ matching_gt_boxes[..., 2], matching_gt_boxes[..., 3] # [b,16,16,5] x2, y2, w2, h2 = pred_xy[..., 0], pred_xy[..., 1], pred_wh[..., 0], pred_wh[..., 1] ious = compute_iou(x1, y1, w1, h1, x2, y2, w2, h2) # [b,16,16,5,1] ious = tf.expand_dims(ious, axis=-1) # [b,16,16,5,1] pred_conf = tf.sigmoid(y_pred[..., 4:5]) # [b,16,16,5,2] => [b,16,16,5, 1, 2] pred_xy = tf.expand_dims(pred_xy, axis=4) # [b,16,16,5,2] => [b,16,16,5, 1, 2] pred_wh = tf.expand_dims(pred_wh, axis=4) pred_wh_half = pred_wh / 2. pred_xymin = pred_xy - pred_wh_half pred_xymax = pred_xy + pred_wh_half # [b, 40, 5] => [b, 1, 1, 1, 40, 5] true_boxes_grid = tf.reshape(gt_boxes_grid, [ gt_boxes_grid.shape[0], 1, 1, 1, gt_boxes_grid.shape[1], gt_boxes_grid.shape[2] ]) true_xy = true_boxes_grid[..., 0:2] true_wh = true_boxes_grid[..., 2:4] true_wh_half = true_wh / 2. true_xymin = true_xy - true_wh_half true_xymax = true_xy + true_wh_half # predxymin, predxymax, true_xymin, true_xymax # [b,16,16,5,1,2] vs [b,1,1,1,40,2]=> [b,16,16,5,40,2] intersectxymin = tf.maximum(pred_xymin, true_xymin) # [b,16,16,5,1,2] vs [b,1,1,1,40,2]=> [b,16,16,5,40,2] intersectxymax = tf.minimum(pred_xymax, true_xymax) # [b,16,16,5,40,2] intersect_wh = tf.maximum(intersectxymax - intersectxymin, 0.) # [b,16,16,5,40] * [b,16,16,5,40]=>[b,16,16,5,40] intersect_area = intersect_wh[..., 0] * intersect_wh[..., 1] # [b,16,16,5,1] pred_area = pred_wh[..., 0] * pred_wh[..., 1] # [b,1,1,1,40] true_area = true_wh[..., 0] * true_wh[..., 1] # [b,16,16,5,1]+[b,1,1,1,40]-[b,16,16,5,40]=>[b,16,16,5,40] union_area = pred_area + true_area - intersect_area # [b,16,16,5,40] iou_score = intersect_area / union_area # [b,16,16,5] best_iou = tf.reduce_max(iou_score, axis=4) # [b,16,16,5,1] best_iou = tf.expand_dims(best_iou, axis=-1) nonobj_detection = tf.cast(best_iou < 0.6, tf.float32) nonobj_mask = nonobj_detection * (1 - detector_mask) # nonobj counter n_nonobj = tf.reduce_sum(tf.cast(nonobj_mask > 0., tf.float32)) nonobj_loss = tf.reduce_sum(nonobj_mask * tf.square(-pred_conf)) \ / (n_nonobj + 1e-6) obj_loss = tf.reduce_sum(detector_mask * tf.square(ious - pred_conf)) \ / (n_detector_mask + 1e-6) loss = coord_loss + class_loss + nonobj_loss + 5 * obj_loss return loss, [nonobj_loss + 5 * obj_loss, class_loss, coord_loss]
def yolo_loss(y_true, y_pred): #tf.print("true", y_true.shape) # 1. transform all pred outputs # y_pred: (batch_size, grid_y, grid_x, anchors, (x, y, w, h, obj, ...cls)) pred_box, pred_obj, pred_class, pred_xywh = yolo_boxes( y_pred, anchors, classes) pred_xy = pred_xywh[..., 0:2] pred_wh = pred_xywh[..., 2:4] #tf.print("before", y_pred[...][0, tf.random.uniform(shape=[], minval=0, maxval=16, dtype=tf.int64), tf.random.uniform(shape=[], minval=0, maxval=16, dtype=tf.int64), tf.random.uniform(shape=[], minval=0, maxval=3, dtype=tf.int64), :]) # 2. transform all true outputs # y_true: (batch_size, grid_y, grid_x, anchors, (x1, y1, x2, y2, obj, cls)) true_box, true_obj, true_class_idx = tf.split(y_true, (4, 1, 1), axis=-1) true_xy = (true_box[..., 0:2] + true_box[..., 2:4]) / 2 true_wh = true_box[..., 2:4] - true_box[..., 0:2] # give higher weights to small boxes box_loss_scale = 2 - true_wh[..., 0] * true_wh[..., 1] # 3. inverting the pred box equations grid_size = tf.shape(y_true)[1:3] grid = tf.meshgrid(tf.range(grid_size[1]), tf.range(grid_size[0])) grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2) true_xy = true_xy * tf.cast((grid_size[1], grid_size[0]), tf.float32) - \ tf.cast(grid, tf.float32) true_wh = tf.math.log(true_wh / anchors) true_wh = tf.where(tf.math.is_inf(true_wh), tf.ones_like(true_wh) * -20, true_wh) # 4. calculate all masks obj_mask = tf.squeeze(true_obj, -1) mask = tf.not_equal(obj_mask, 0) not_mask = tf.logical_not(mask) summarize = 20 #tf.print("a", pred_xy.shape) #tf.print("b", obj_mask.shape) #tf.print("c", mask.shape) #tf.print("d", tf.boolean_mask(true_obj, mask), summarize=summarize) #tf.print("e", tf.boolean_mask(pred_obj, mask), summarize=summarize) # ignore false positive when iou is over threshold best_iou = tf.map_fn( lambda x: tf.reduce_max(broadcast_iou( x[0], tf.boolean_mask(x[1], tf.cast(x[2], tf.bool))), axis=-1), (pred_box, true_box, obj_mask), tf.float32) ignore_mask = tf.cast(best_iou < ignore_thresh, tf.float32) # 5. calculate all losses xy_loss = obj_mask * box_loss_scale * \ tf.reduce_sum(tf.square(true_xy - pred_xy), axis=-1) wh_loss = obj_mask * box_loss_scale * \ tf.reduce_sum(tf.square(true_wh - pred_wh), axis=-1) #tf.print("losses 1", tf.math.reduce_sum(xy_loss)) #tf.print("losses 2", tf.math.reduce_sum(wh_loss)) #tf.print("true_obj shape", true_obj.shape, true_obj.dtype) #tf.print("pred_obj shape", pred_obj.shape, pred_obj.dtype) obj_loss = binary_crossentropy(true_obj, pred_obj) #tf.print("obj_loss shape", obj_loss.shape) #tf.print("f", tf.boolean_mask(obj_loss, mask), summarize=summarize) obj_loss = obj_mask * obj_loss + \ (1 - obj_mask) * ignore_mask * obj_loss #tf.print("losses 3", tf.math.reduce_sum(obj_loss)) #tf.print("g", tf.boolean_mask(y_pred, mask), summarize=summarize) #tf.print("h", tf.boolean_mask(y_pred, not_mask), summarize=summarize) #tf.print("g_true", tf.boolean_mask(y_true, mask), summarize=summarize) #tf.print("h_true", tf.boolean_mask(y_true, not_mask), summarize=summarize) #tf.cond(tf.logical_or(tf.math.is_nan(tf.math.reduce_sum(obj_loss)), tf.math.is_inf(tf.math.reduce_sum(obj_loss))), lambda: [tf.print(y_pred), y_pred][-1], lambda: y_pred) # TODO: use binary_crossentropy instead class_loss = obj_mask * sparse_categorical_crossentropy( true_class_idx, pred_class) # 6. sum over (batch, gridx, gridy, anchors) => (batch, 1) xy_loss = tf.reduce_sum(xy_loss, axis=(1, 2, 3)) wh_loss = tf.reduce_sum(wh_loss, axis=(1, 2, 3)) obj_loss = tf.reduce_sum(obj_loss, axis=(1, 2, 3)) class_loss = tf.reduce_sum(class_loss, axis=(1, 2, 3)) loss = xy_loss + wh_loss + obj_loss + class_loss loss_mask = tf.logical_or(tf.math.is_nan(loss), tf.math.is_nan(loss)) #tf.print("xy_loss", tf.boolean_mask(xy_loss, loss_mask)) #tf.print("wh_loss", tf.boolean_mask(wh_loss, loss_mask)) #tf.print("obj_loss", tf.boolean_mask(obj_loss, loss_mask)) #tf.print("class_loss", tf.boolean_mask(class_loss, loss_mask)) #tf.print("loss", tf.boolean_mask(loss, loss_mask)) #tf.print("non entries", tf.boolean_mask(y_pred, loss_mask)) #tf.print("tru entries", tf.boolean_mask(y_true, loss_mask)) #tf.print("LOSS", xy_loss + wh_loss + obj_loss + class_loss) return xy_loss + wh_loss + obj_loss + class_loss
def sparse_cat_loss(y_true, y_pred): #wrapper return sparse_categorical_crossentropy( y_true, y_pred, from_logits=True) #for one-hot-encoded features
def yolo_loss(y_true, y_pred): # Part 1 - Transform all pred outputs: # y_pred: (batch_size, grid, grid, anchors, (x, y, w, h, obj, ...cls)) pred_box, pred_obj, pred_class, pred_xywh = yolo_boxes(pred=y_pred, anchors=anchors, classes=classes) # Split the predicted box shape lines: pred_xy = pred_xywh[..., 0:2] pred_wh = pred_xywh[..., 2:4] # Part 2 - Transform all true outputs # y_true: (batch_size, grid, grid, anchors, (x1, y1, x2, y2, obj, cls)) true_box, true_obj, true_class_idx = tf.split(value=y_true, num_or_size_splits=(4, 1, 1), axis=-1) # Split the Ground truth box shape lines: true_xy = (true_box[..., 0:2] + true_box[..., 2:4]) / 2 true_wh = true_box[..., 2:4] - true_box[..., 0:2] # Allow for higher weights to small boxes: box_loss_scale = 2 - true_wh[..., 0] * true_wh[..., 1] # Part 3 - Inverting the pred box equations: grid_size = tf.shape(y_true)[1] grid = tf.meshgrid(tf.range(grid_size), tf.range(grid_size)) grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2) # Convert true_xy to match grid size: true_xy = true_xy * tf.cast(grid_size, tf.float32) - tf.cast( grid, tf.float32) true_wh = tf.math.log(true_wh / anchors) true_wh = tf.where(tf.math.is_inf(true_wh), tf.zeros_like(true_wh), true_wh) # Part 4 - Compute all masks: obj_mask = tf.squeeze(input=true_obj, axis=-1) # Ignore when IoU is over the threshold stated: true_box_flat = tf.boolean_mask(tensor=true_box, mask=tf.cast(obj_mask, tf.bool)) # Find the Best IoU: best_iou = tf.reduce_max(intersectionOverUnion(box1=pred_box, box2=true_box_flat), axis=-1) # Ignore IoU: ignore_mask = tf.cast(best_iou < ignore_threshold, tf.float32) # Part 5 - Compute all losses: xy_loss = obj_mask * box_loss_scale * tf.reduce_sum( input_tensor=tf.square(true_xy - pred_xy), axis=-1) wh_loss = obj_mask * box_loss_scale * tf.reduce_sum( input_tensor=tf.square(true_wh - pred_wh), axis=-1) # Using binary_crossentropy for the object loss: obj_loss = binary_crossentropy(y_true=true_obj, y_pred=pred_obj) obj_loss = obj_mask * obj_loss + (1 - obj_mask) * ignore_mask * obj_loss # Using sparse_categorical_crossentropy instead for the classes loss: class_loss = obj_mask * sparse_categorical_crossentropy( y_true=true_class_idx, y_pred=pred_class) # Part 6 - Sum over (batch, gridx, gridy, anchors) => (batch, 1) xy_loss = tf.reduce_sum(xy_loss, axis=(1, 2, 3)) wh_loss = tf.reduce_sum(wh_loss, axis=(1, 2, 3)) obj_loss = tf.reduce_sum(obj_loss, axis=(1, 2, 3)) class_loss = tf.reduce_sum(class_loss, axis=(1, 2, 3)) return xy_loss + wh_loss + obj_loss + class_loss
def sparse_categorical_crossentropy(y_true, y_pred): y_true = tf.cast(y_true, 'float32') y_pred = tf.cast(y_pred, 'float32') return losses.sparse_categorical_crossentropy(y_true, y_pred)