def call(self, inputs, output_shape=None): updates, mask = inputs[0], inputs[1] with tf.compat.v1.variable_scope(self.name): mask = K.cast(mask, "int32") input_shape = tf.shape(updates, out_type="int32") # calculation new shape if output_shape is None: output_shape = ( input_shape[0], input_shape[1] * self.size[0], input_shape[2] * self.size[1], input_shape[3], ) self.output_shape1 = output_shape # calculation indices for batch, height, width and feature maps one_like_mask = K.ones_like(mask, dtype="int32") batch_shape = K.concatenate([[input_shape[0]], [1], [1], [1]], axis=0) batch_range = K.reshape(tf.range(output_shape[0], dtype="int32"), shape=batch_shape) b = one_like_mask * batch_range y = mask // (output_shape[2] * output_shape[3]) x = (mask // output_shape[3]) % output_shape[2] feature_range = tf.range(output_shape[3], dtype="int32") f = one_like_mask * feature_range # transpose indices & reshape update values to one dimension updates_size = tf.size(updates) indices = K.transpose( K.reshape(K.stack([b, y, x, f]), [4, updates_size])) values = K.reshape(updates, [updates_size]) ret = tf.scatter_nd(indices, values, output_shape) return ret
def call(self, QKVs): """Core logic of multi-head self attention. Args: QKVs (list): inputs of multi-head self attention i.e. query, key and value. Returns: object: ouput tensors. """ if len(QKVs) == 3: Q_seq, K_seq, V_seq = QKVs Q_len, V_len = None, None elif len(QKVs) == 5: Q_seq, K_seq, V_seq, Q_len, V_len = QKVs Q_seq = K.dot(Q_seq, self.WQ) Q_seq = K.reshape(Q_seq, shape=(-1, K.shape(Q_seq)[1], self.multiheads, self.head_dim)) Q_seq = K.permute_dimensions(Q_seq, pattern=(0, 2, 1, 3)) K_seq = K.dot(K_seq, self.WK) K_seq = K.reshape(K_seq, shape=(-1, K.shape(K_seq)[1], self.multiheads, self.head_dim)) K_seq = K.permute_dimensions(K_seq, pattern=(0, 2, 1, 3)) V_seq = K.dot(V_seq, self.WV) V_seq = K.reshape(V_seq, shape=(-1, K.shape(V_seq)[1], self.multiheads, self.head_dim)) V_seq = K.permute_dimensions(V_seq, pattern=(0, 2, 1, 3)) A = einsum("abij, abkj -> abik", Q_seq, K_seq) / K.sqrt( K.cast(self.head_dim, dtype="float32")) A = K.permute_dimensions( A, pattern=(0, 3, 2, 1) ) # A.shape=[batch_size,K_sequence_length,Q_sequence_length,self.multiheads] A = self.Mask(A, V_len, "add") A = K.permute_dimensions(A, pattern=(0, 3, 2, 1)) if self.mask_right: ones = K.ones_like(A[:1, :1]) lower_triangular = K.tf.matrix_band_part(ones, num_lower=-1, num_upper=0) mask = (ones - lower_triangular) * 1e12 A = A - mask A = K.softmax(A) O_seq = einsum("abij, abjk -> abik", A, V_seq) O_seq = K.permute_dimensions(O_seq, pattern=(0, 2, 1, 3)) O_seq = K.reshape(O_seq, shape=(-1, K.shape(O_seq)[1], self.output_dim)) O_seq = self.Mask(O_seq, Q_len, "mul") return O_seq
def yolo_eval(yolo_outputs, anchors, num_classes, image_shape, max_boxes=80, score_threshold=.5, iou_threshold=.5): """Evaluate YOLO model on given input and return filtered boxes.""" input_shape = K.shape(yolo_outputs)[1:3] * grid_size_multiplier boxes = [] box_scores = [] polygons = [] for l in range(1): _boxes, _box_scores, _polygons = yolo_boxes_and_scores( yolo_outputs, anchors[anchor_mask[l]], num_classes, input_shape, image_shape) boxes.append(_boxes) box_scores.append(_box_scores) polygons.append(_polygons) boxes = K.concatenate(boxes, axis=0) box_scores = K.concatenate(box_scores, axis=0) polygons = K.concatenate(polygons, axis=0) mask = box_scores >= score_threshold box_scores >= score_threshold max_boxes_tensor = K.constant(max_boxes, dtype='int32') boxes_ = [] scores_ = [] classes_ = [] polygons_ = [] for c in range(num_classes): # TODO: use keras backend instead of tf. class_boxes = tf.boolean_mask(boxes, mask[:, c]) class_polygons = tf.boolean_mask(polygons, mask[:, c]) class_box_scores = tf.boolean_mask(box_scores[:, c], mask[:, c]) nms_index = tf.image.non_max_suppression(class_boxes, class_box_scores, max_boxes_tensor, iou_threshold=iou_threshold) class_boxes = K.gather(class_boxes, nms_index) class_box_scores = K.gather(class_box_scores, nms_index) class_polygons = K.gather(class_polygons, nms_index) classes = K.ones_like(class_box_scores, 'int32') * c boxes_.append(class_boxes) scores_.append(class_box_scores) classes_.append(classes) polygons_.append(class_polygons) polygons_ = K.concatenate(polygons_, axis=0) boxes_ = K.concatenate(boxes_, axis=0) scores_ = K.concatenate(scores_, axis=0) classes_ = K.concatenate(classes_, axis=0) return boxes_, scores_, classes_, polygons_
def yolo_eval(yolo_outputs, anchors, num_classes, image_shape, max_boxes=20, score_threshold=.6, iou_threshold=.5): """Evaluate YOLO model on given input and return filtered boxes.""" num_layers = len(yolo_outputs) anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] if num_layers == 3 else [[ 3, 4, 5 ], [1, 2, 3]] # default setting input_shape = K.shape(yolo_outputs[0])[1:3] * 32 boxes = [] box_scores = [] for l in range(num_layers): _boxes, _box_scores = yolo_boxes_and_scores(yolo_outputs[l], anchors[anchor_mask[l]], num_classes, input_shape, image_shape) boxes.append(_boxes) box_scores.append(_box_scores) boxes = K.concatenate(boxes, axis=0) box_scores = K.concatenate(box_scores, axis=0) mask = box_scores >= score_threshold max_boxes_tensor = K.constant(max_boxes, dtype='int32') boxes_ = [] scores_ = [] classes_ = [] for c in range(num_classes): # TODO: use keras backend instead of tf. class_boxes = tf.boolean_mask(boxes, mask[:, c]) class_box_scores = tf.boolean_mask(box_scores[:, c], mask[:, c]) nms_index = tf.image.non_max_suppression(class_boxes, class_box_scores, max_boxes_tensor, iou_threshold=iou_threshold) class_boxes = K.gather(class_boxes, nms_index) class_box_scores = K.gather(class_box_scores, nms_index) classes = K.ones_like(class_box_scores, 'int32') * c boxes_.append(class_boxes) scores_.append(class_box_scores) classes_.append(classes) boxes_ = K.concatenate(boxes_, axis=0) scores_ = K.concatenate(scores_, axis=0) classes_ = K.concatenate(classes_, axis=0) return boxes_, scores_, classes_