def yolo_head(feats, anchors, num_classes, input_shape, calc_loss=False): """Convert final layer features to bounding box parameters.""" num_anchors = anchors_per_level # Reshape to batch, height, width, num_anchors, box_params. anchors_tensor = K.reshape(K.constant(anchors), [1, 1, 1, num_anchors, 2]) grid_shape = K.shape(feats)[1:3] # height, width grid_y = K.tile( tf.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1], name='yolo_head/tile/reshape/grid_y'), [1, grid_shape[1], 1, 1]) grid_x = K.tile( tf.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1], name='yolo_head/tile/reshape/grid_x'), [grid_shape[0], 1, 1, 1]) grid = tf.concat([grid_x, grid_y], axis=-1, name='yolo_head/concatenate/grid') grid = K.cast(grid, K.dtype(feats)) feats = tf.reshape(feats, [ -1, grid_shape[0], grid_shape[1], num_anchors, num_classes + 5 + NUM_ANGLES3 ], name='yolo_head/reshape/feats') # Adjust predictions to each spatial grid point and anchor size. box_xy = (K.sigmoid(feats[..., :2]) + grid) / K.cast( grid_shape[..., ::-1], K.dtype(feats)) box_wh = K.exp(feats[..., 2:4]) * anchors_tensor / K.cast( input_shape[..., ::-1], K.dtype(feats)) box_confidence = K.sigmoid(feats[..., 4:5]) box_class_probs = K.sigmoid(feats[..., 5:5 + num_classes]) polygons_confidence = K.sigmoid(feats[..., 5 + num_classes + 2:5 + num_classes + NUM_ANGLES3:3]) polygons_x = K.exp(feats[..., 5 + num_classes:num_classes + 5 + NUM_ANGLES3:3]) dx = K.square(anchors_tensor[..., 0:1] / 2) dy = K.square(anchors_tensor[..., 1:2] / 2) d = K.cast(K.sqrt(dx + dy), K.dtype(polygons_x)) a = K.pow(input_shape[..., ::-1], 2) a = K.cast(a, K.dtype(feats)) b = K.sum(a) diagonal = K.cast(K.sqrt(b), K.dtype(feats)) polygons_x = polygons_x * d / diagonal polygons_y = feats[..., 5 + num_classes + 1:num_classes + 5 + NUM_ANGLES3:3] polygons_y = K.sigmoid(polygons_y) if calc_loss == True: return grid, feats, box_xy, box_wh, polygons_confidence return box_xy, box_wh, box_confidence, box_class_probs, polygons_x, polygons_y, polygons_confidence
def euclidean_distance(vects): ''' Computes the euclidean distances between vects[0] and vects[1] ''' x, y = vects return K.sqrt( K.maximum(K.sum(K.square(x - y), axis=1, keepdims=True), K.epsilon()))
def pearson_correlation_metric_fn( y_true: tf.Tensor, y_pred: tf.Tensor, ) -> tf.Tensor: """ Pearson correlation metric function. https://github.com/WenYanger/Keras_Metrics Args: y_true (tf.Tensor): y_true y_pred (tf.Tensor): y_pred Returns: tf.contrib.metrics: pearson correlation """ x = y_true y = y_pred mx = K.mean(x, axis=0) my = K.mean(y, axis=0) xm, ym = x - mx, y - my r_num = K.sum(xm * ym) x_square_sum = K.sum(xm * xm) y_square_sum = K.sum(ym * ym) r_den = K.sqrt(x_square_sum * y_square_sum) + 1e-12 r = r_num / r_den return K.mean(r)
def call(self, QKVs): """Core logic of multi-head self attention. Args: QKVs (list): inputs of multi-head self attention i.e. query, key and value. Returns: object: ouput tensors. """ if len(QKVs) == 3: Q_seq, K_seq, V_seq = QKVs Q_len, V_len = None, None elif len(QKVs) == 5: Q_seq, K_seq, V_seq, Q_len, V_len = QKVs Q_seq = K.dot(Q_seq, self.WQ) Q_seq = K.reshape(Q_seq, shape=(-1, K.shape(Q_seq)[1], self.multiheads, self.head_dim)) Q_seq = K.permute_dimensions(Q_seq, pattern=(0, 2, 1, 3)) K_seq = K.dot(K_seq, self.WK) K_seq = K.reshape(K_seq, shape=(-1, K.shape(K_seq)[1], self.multiheads, self.head_dim)) K_seq = K.permute_dimensions(K_seq, pattern=(0, 2, 1, 3)) V_seq = K.dot(V_seq, self.WV) V_seq = K.reshape(V_seq, shape=(-1, K.shape(V_seq)[1], self.multiheads, self.head_dim)) V_seq = K.permute_dimensions(V_seq, pattern=(0, 2, 1, 3)) A = einsum("abij, abkj -> abik", Q_seq, K_seq) / K.sqrt( K.cast(self.head_dim, dtype="float32")) A = K.permute_dimensions( A, pattern=(0, 3, 2, 1) ) # A.shape=[batch_size,K_sequence_length,Q_sequence_length,self.multiheads] A = self.Mask(A, V_len, "add") A = K.permute_dimensions(A, pattern=(0, 3, 2, 1)) if self.mask_right: ones = K.ones_like(A[:1, :1]) lower_triangular = K.tf.matrix_band_part(ones, num_lower=-1, num_upper=0) mask = (ones - lower_triangular) * 1e12 A = A - mask A = K.softmax(A) O_seq = einsum("abij, abjk -> abik", A, V_seq) O_seq = K.permute_dimensions(O_seq, pattern=(0, 2, 1, 3)) O_seq = K.reshape(O_seq, shape=(-1, K.shape(O_seq)[1], self.output_dim)) O_seq = self.Mask(O_seq, Q_len, "mul") return O_seq
def get_log_probability_density(pred, y): mu_and_sigma = pred mu = mu_and_sigma[:, :2] sigma = mu_and_sigma[:, 2:] variance = K.square(sigma) pdf = 1. / K.sqrt(2. * np.pi * variance) * K.exp(-K.square(y - mu) / (2. * variance)) log_pdf = K.log(pdf + K.epsilon()) return log_pdf
def squared_distance(input_x, input_y=None, weight=None): """Calculates the pairwise distance between points in X and Y. Args: input_x: n x d matrix input_y: m x d matrix weight: affinity n x m -- if provided, we normalize the distance Returns: n x m matrix of all pairwise squared Euclidean distances """ if input_y is None: input_y = input_x sum_dimensions = list(range(2, K.ndim(input_x) + 1)) input_x = K.expand_dims(input_x, axis=1) if weight is not None: # if weight provided, we normalize input_x and input_y by weight d_diag = K.expand_dims(K.sqrt(K.sum(weight, axis=1)), axis=1) input_x /= d_diag input_y /= d_diag squared_difference = K.square(input_x - input_y) distance = K.sum(squared_difference, axis=sum_dimensions) return distance
def squared_distance(X, Y=None, W=None): ''' Calculates the pairwise distance between points in X and Y X: n x d matrix Y: m x d matrix W: affinity -- if provided, we normalize the distance returns: n x m matrix of all pairwise squared Euclidean distances ''' if Y is None: Y = X # distance = squaredDistance(X, Y) sum_dimensions = list(range(2, K.ndim(X) + 1)) X = K.expand_dims(X, axis=1) if W is not None: # if W provided, we normalize X and Y by W D_diag = K.expand_dims(K.sqrt(K.sum(W, axis=1)), axis=1) X /= D_diag Y /= D_diag squared_difference = K.square(X - Y) distance = K.sum(squared_difference, axis=sum_dimensions) return distance
def yolo_loss(args, anchors, num_classes, ignore_thresh=.5): """Return yolo_loss tensor Parameters ---------- yolo_outputs: list of tensor, the output of yolo_body or tiny_yolo_body y_true: list of array, the output of preprocess_true_boxes anchors: array, shape=(N, 2), wh num_classes: integer ignore_thresh: float, the iou threshold whether to ignore object confidence loss Returns ------- loss: tensor, shape=(1,) """ num_layers = 1 yolo_outputs = args[:num_layers] y_true = args[num_layers:] g_y_true = y_true input_shape = K.cast( K.shape(yolo_outputs[0])[1:3] * grid_size_multiplier, K.dtype(y_true[0])) grid_shapes = [ K.cast(K.shape(yolo_outputs[l])[1:3], K.dtype(y_true[0])) for l in range(num_layers) ] loss = 0 m = K.shape(yolo_outputs[0])[0] # batch size, tensor mf = K.cast(m, K.dtype(yolo_outputs[0])) for layer in range(num_layers): object_mask = y_true[layer][..., 4:5] vertices_mask = y_true[layer][..., 5 + num_classes + 2:5 + num_classes + NUM_ANGLES3:3] true_class_probs = y_true[layer][..., 5:5 + num_classes] grid, raw_pred, pred_xy, pred_wh, pol_cnf = yolo_head( yolo_outputs[layer], anchors[anchor_mask[layer]], num_classes, input_shape, calc_loss=True) pred_box = K.concatenate([pred_xy, pred_wh]) raw_true_xy = y_true[layer][..., :2] * grid_shapes[layer][ ..., ::-1] - grid raw_true_polygon0 = y_true[layer][..., 5 + num_classes:5 + num_classes + NUM_ANGLES3] raw_true_wh = K.log(y_true[layer][..., 2:4] / anchors[anchor_mask[layer]] * input_shape[..., ::-1]) raw_true_wh = K.switch(object_mask, raw_true_wh, K.zeros_like(raw_true_wh)) # avoid log(0)=-inf raw_true_polygon_x = raw_true_polygon0[..., ::3] raw_true_polygon_y = raw_true_polygon0[..., 1::3] dx = K.square(anchors[anchor_mask[layer]][..., 0:1] / 2) dy = K.square(anchors[anchor_mask[layer]][..., 1:2] / 2) d = K.cast(K.sqrt(dx + dy), K.dtype(raw_true_polygon_x)) diagonal = K.sqrt( K.pow(input_shape[..., ::-1][0], 2) + K.pow(input_shape[..., ::-1][1], 2)) raw_true_polygon_x = K.log(raw_true_polygon_x / d * diagonal) raw_true_polygon_x = K.switch(vertices_mask, raw_true_polygon_x, K.zeros_like(raw_true_polygon_x)) box_loss_scale = 2 - y_true[layer][..., 2:3] * y_true[layer][..., 3:4] # Find ignore mask, iterate over each of batch. ignore_mask = tf.TensorArray(K.dtype(y_true[0]), size=1, dynamic_size=True) object_mask_bool = K.cast(object_mask, 'bool') def loop_body(b, ignore_mask): true_box = tf.boolean_mask(y_true[layer][b, ..., 0:4], object_mask_bool[b, ..., 0]) iou = box_iou(pred_box[b], true_box) best_iou = K.max(iou, axis=-1) ignore_mask = ignore_mask.write( b, K.cast(best_iou < ignore_thresh, K.dtype(true_box))) return b + 1, ignore_mask _, ignore_mask = tf.while_loop(lambda b, *args: b < m, loop_body, [0, ignore_mask]) ignore_mask = ignore_mask.stack() ignore_mask = K.expand_dims(ignore_mask, -1) # K.binary_crossentropy is helpful to avoid exp overflow. xy_loss = object_mask * box_loss_scale * K.binary_crossentropy( raw_true_xy, raw_pred[..., 0:2], from_logits=True) wh_loss = object_mask * box_loss_scale * 0.5 * K.square( raw_true_wh - raw_pred[..., 2:4]) confidence_loss = object_mask * K.binary_crossentropy( object_mask, raw_pred[..., 4:5], from_logits=True ) + (1 - object_mask) * K.binary_crossentropy( object_mask, raw_pred[..., 4:5], from_logits=True) * ignore_mask class_loss = object_mask * K.binary_crossentropy( true_class_probs, raw_pred[..., 5:5 + num_classes], from_logits=True) polygon_loss_x = object_mask * vertices_mask * box_loss_scale * 0.5 * K.square( raw_true_polygon_x - raw_pred[..., 5 + num_classes:5 + num_classes + NUM_ANGLES3:3]) polygon_loss_y = object_mask * vertices_mask * box_loss_scale * K.binary_crossentropy( raw_true_polygon_y, raw_pred[..., 5 + num_classes + 1:5 + num_classes + NUM_ANGLES3:3], from_logits=True) vertices_confidence_loss = object_mask * K.binary_crossentropy( vertices_mask, raw_pred[..., 5 + num_classes + 2:5 + num_classes + NUM_ANGLES3:3], from_logits=True) xy_loss = K.sum(xy_loss) / mf wh_loss = K.sum(wh_loss) / mf class_loss = K.sum(class_loss) / mf confidence_loss = K.sum(confidence_loss) / mf vertices_confidence_loss = K.sum(vertices_confidence_loss) / mf polygon_loss = K.sum(polygon_loss_x) / mf + K.sum(polygon_loss_y) / mf diou_loss = K.sum( object_mask * box_loss_scale * (1 - box_diou(pred_box, y_true[layer][..., 0:4]))) / mf loss += (xy_loss + wh_loss + confidence_loss + class_loss + 0.2 * polygon_loss + 0.2 * vertices_confidence_loss) / ( K.sum(object_mask) + 1) * mf return loss
def root_mean_squared_error(y_true, y_pred): return K.sqrt(K.mean(K.square(y_pred - y_true)))
def __euclidean(x, y): diff = tf.subtract(x, y) square_diff = K.pow(diff, 2) d = K.sqrt(K.sum(square_diff, axis=-1)) return d