def call(self, inputs, training=None, mask=None):
x = self.conv1(inputs)
x = self.bn1(x, training=training)
x = h_swish(x)
x = self.bneck1(x, training=training)
x = self.bneck2(x, training=training)
x = self.bneck3(x, training=training)
x = self.bneck4(x, training=training)
x = self.bneck5(x, training=training)
x = self.bneck6(x, training=training)
x = self.bneck7(x, training=training)
x = self.bneck8(x, training=training)
x = self.bneck9(x, training=training)
x = self.bneck10(x, training=training)
x = self.bneck11(x, training=training)
x = self.bneck12(x, training=training)
x = self.bneck13(x, training=training)
x = self.bneck14(x, training=training)
x = self.bneck15(x, training=training)
x = self.conv2(x)
x = self.bn2(x, training=training)
x = h_swish(x)
x = self.avgpool(x)
x = self.conv3(x)
x = h_swish(x)
x = self.conv4(x)
return x
def call(self, inputs, training=None, mask=None):
x = self.conv1(inputs)
x = self.bn1(x, training=training)
x = h_swish(x)
x = self.bneck1(x, training=training)
x = self.bneck2(x, training=training)
x = self.bneck3(x, training=training)
x = self.bneck4(x, training=training)
x = self.bneck5(x, training=training)
x = self.bneck6(x, training=training)
x = self.bneck7(x, training=training)
x = self.bneck8(x, training=training)
x = self.bneck9(x, training=training)
x = self.bneck10(x, training=training)
x = self.bneck11(x, training=training)
x = self.conv2(x)
x = self.bn2(x, training=training)
x = h_swish(x)
# ValueError: Negative dimension size caused by subtracting 7 from 1 for '{{node average_pooling2d/AvgPool}} = AvgPool[T=DT_FLOAT, data_format="NHWC", ksize=[1, 7, 7, 1], padding="VALID", strides=[1, 1, 1, 1]](mul_1)' with input shapes: [?,1,1,576].
# 下面行报错的原因是因为输入图片的尺寸太小了，需要224，224，3的图片或者别的尺寸的图片，需要调整图片大小
x = self.avgpool(x)
x = self.conv3(x)
x = h_swish(x)
x = self.conv4(x)
return x
def _build_backbone(self, x):
x = Conv2D(filters=16, kernel_size=(3, 3), strides=2, padding="same")(x) # 256, 2
x = BatchNormalization(name='first_bn', epsilon=1e-5)(x)
x = h_swish(x)
x = BottleNeck(in_size=16, exp_size=16, out_size=16, s=1, is_se_existing=False, NL="RE", k=3)(x) # 256
x = BottleNeck(in_size=16, exp_size=64, out_size=24, s=2, is_se_existing=False, NL="RE", k=3)(x) # 128, 4
s_4 = BottleNeck(in_size=24, exp_size=72, out_size=24, s=1, is_se_existing=False, NL="RE", k=3)(x) # 128
x = BottleNeck(in_size=24, exp_size=72, out_size=40, s=2, is_se_existing=True, NL="RE", k=5)(s_4) # 64, 8
x = BottleNeck(in_size=40, exp_size=120, out_size=40, s=1, is_se_existing=True, NL="RE", k=5)(x)
s_8 = BottleNeck(in_size=40, exp_size=120, out_size=40, s=1, is_se_existing=True, NL="RE", k=5)(x)
x = BottleNeck(in_size=40, exp_size=240, out_size=80, s=2, is_se_existing=False, NL="HS", k=3)(s_8) # 32
x = BottleNeck(in_size=80, exp_size=200, out_size=80, s=1, is_se_existing=False, NL="HS", k=3)(x)
x = BottleNeck(in_size=80, exp_size=184, out_size=80, s=1, is_se_existing=False, NL="HS", k=3)(x)
x = BottleNeck(in_size=80, exp_size=184, out_size=80, s=1, is_se_existing=False, NL="HS", k=3)(x)
x = BottleNeck(in_size=80, exp_size=480, out_size=112, s=1, is_se_existing=True, NL="HS", k=3)(x)
s_16 = BottleNeck(in_size=112, exp_size=672, out_size=112, s=1, is_se_existing=True, NL="HS", k=3)(x)
x = BottleNeck(in_size=112, exp_size=672, out_size=160, s=2, is_se_existing=True, NL="HS", k=5)(s_16) # 16
x = BottleNeck(in_size=160, exp_size=960, out_size=160, s=1, is_se_existing=True, NL="HS", k=5)(x)
s_32 = BottleNeck(in_size=160, exp_size=960, out_size=160, s=1, is_se_existing=True, NL="HS", k=5)(x)
# x = Conv2D(filters=960, kernel_size=(1, 1), strides=1, padding="same")(x)
# x = BatchNormalization(epsilon=1e-5)(x)
# s_32 = Activation('relu')(x)
return s_4, s_8, s_16, s_32
def _build_graph(self):
x = self.conv1(self.images)
x = self.bn1(x)
x = h_swish(x) # 208...112
x = self.bneck1_1(x) # 112
x = self.bneck1_2(x) # 56
self.s_4 = self.bneck2_1(x) # 56
x = self.bneck2_2(self.s_4) # 28
x = self.bneck3_1(x) # 28
self.s_8 = self.bneck3_2(x) # 28
x = self.bneck3_3(self.s_8) # 14
x = self.bneck4_1(x) # 14
x = self.bneck4_2(x) # 14
x = self.bneck4_3(x) # 14
x = self.bneck4_4(x) # 14
self.s_16 = self.bneck4_5(x) # 14
x = self.bneck4_6(self.s_16) # 7
x = self.bneck5_1(x) # 7
self.s_32 = self.bneck5_2(x) # 7
feature_map = self._fusion_feature()
keypoints, preg, fpn = self._detect_head(feature_map)
if self.mode == 'train':
center_loss = CentLoss(self.batch_size, self.num_classes, self.loss_decay, self.stride)\
([keypoints, preg, fpn, self.ground_truth, self.mask_ground_truth, self.scalar_y, self.scalar_x])
inputs = [self.images, self.ground_truth, self.mask_ground_truth, self.scalar_y, self.scalar_x]
outputs = [keypoints, preg, fpn, center_loss]
else:
pshape = [self.image_size/self.stride, self.image_size/self.stride]
h = tf.range(0., tf.cast(pshape[0], tf.float32), dtype=tf.float32)
w = tf.range(0., tf.cast(pshape[1], tf.float32), dtype=tf.float32)
# shape of coordinate equals [h_y_num, w_x_mun]
[meshgrid_x, meshgrid_y] = tf.meshgrid(w, h)
meshgrid_y = tf.expand_dims(meshgrid_y, axis=-1) # [Y, X, -1]
meshgrid_x = tf.expand_dims(meshgrid_x, axis=-1)
# [y, x, 2]
center = tf.concat([meshgrid_y, meshgrid_x], axis=-1)
# [batch_size, y, x, class_num] activate feature maps
# [y, x, class_num]
keypoints = tf.sigmoid(keypoints)
fpn = tf.sigmoid(fpn)
preg = preg
# [y, x]
category = tf.argmax(keypoints[0], axis=2, output_type=tf.int32)
# [1, y, x, 1]
max_key = tf.reduce_max(keypoints, axis=2, keepdims=True)
# 3*3 to be peak value
peak_key = self._max_pooling(max_key, 3, 1)
# mask for each peak_point in each 3*3 area, [y, x] (0,1)
mask_key = tf.cast(tf.equal(max_key, peak_key), tf.float32)
# [1, y, x, 1]
mask_key = max_key * mask_key
# [y*x]
scores = tf.reshape(mask_key, [-1])
# [y*x]
class_id = tf.reshape(category, [-1])
# [(y* x), 2]
grid_yx = tf.reshape(center, [-1, 2])
# [(y*x), 4]
bbox_lrtb = tf.reshape(preg, [-1, 4])
# [y*x, 4(y1, x1, y2, x2)]
bbox = tf.concat([grid_yx - bbox_lrtb[..., -2::-2], grid_yx + bbox_lrtb[..., -1::-2]], axis=-1)
select_indices = tf.image.non_max_suppression(bbox, scores, self.top_k_results_output,
self.nms_threshold, score_threshold=self.score_threshold)
# [num_select, ?]
select_scores = tf.gather(scores, select_indices)
select_center = tf.gather(grid_yx, select_indices)
select_class_id = tf.gather(class_id, select_indices)
select_bbox = tf.gather(bbox, select_indices)
# select_lrtb = tf.gather(bbox_lrtb, select_indices)
class_seg = tf.cast(tf.greater(fpn, self.seg_threshold), tf.float32)
select_scores = tf.expand_dims(select_scores, 0)
select_bbox = tf.expand_dims(select_bbox, 0)
select_class_id = tf.expand_dims(select_class_id, 0)
select_center = tf.expand_dims(select_center, 0)
outputs = [select_center, select_scores, select_bbox, select_class_id, class_seg, preg]
inputs = [self.images]
self.CenterNetModel = tf.keras.Model(inputs=inputs, outputs=outputs)
def _build_graph(self):
x = self.conv1(self.images)
x = self.bn1(x)
x = h_swish(x) # 208...112
x = self.bneck1_1(x) # 112
x = self.bneck1_2(x) # 56
self.s_4 = self.bneck2_1(x) # 56
x = self.bneck2_2(self.s_4) # 28
x = self.bneck3_1(x) # 28
self.s_8 = self.bneck3_2(x) # 28
x = self.bneck3_3(self.s_8) # 14
x = self.bneck4_1(x) # 14
x = self.bneck4_2(x) # 14
x = self.bneck4_3(x) # 14
x = self.bneck4_4(x) # 14
self.s_16 = self.bneck4_5(x) # 14
x = self.bneck4_6(self.s_16) # 7
x = self.bneck5_1(x) # 7
self.s_32 = self.bneck5_2(x) # 7
feature_map = self._fusion_feature()
keypoints, preg, fpn = self._detect_head(feature_map)
if self.mode == 'train':
center_loss = CentLoss(self.batch_size, self.num_classes, self.loss_decay, self.stride)\
([keypoints, preg, fpn, self.ground_truth, self.mask_ground_truth, self.scalar_y, self.scalar_x])
inputs = [self.images, self.ground_truth, self.mask_ground_truth, self.scalar_y, self.scalar_x]
outputs = [keypoints, preg, fpn, center_loss]
else:
pshape = [self.image_size/self.stride, self.image_size/self.stride]
h = tf.range(0., tf.cast(pshape[0], tf.float32), dtype=tf.float32)
w = tf.range(0., tf.cast(pshape[1], tf.float32), dtype=tf.float32)
# shape of coordinate equals [h_y_num, w_x_mun]
[meshgrid_x, meshgrid_y] = tf.meshgrid(w, h)
meshgrid_y = tf.expand_dims(meshgrid_y, axis=-1) # [Y, X, -1]
meshgrid_x = tf.expand_dims(meshgrid_x, axis=-1)
# [y, x, 2]
center = tf.concat([meshgrid_y, meshgrid_x], axis=-1)
# [batch_size, y, x, class_num] activate feature maps
# [y, x, class_num]
keypoints = tf.sigmoid(keypoints)
fpn = tf.sigmoid(fpn)
preg = preg
for i in range(self.batch_size):
# # [1, y, x, class_num]
pic_keypoints = tf.expand_dims(keypoints[i], axis=0)
# [1, y, x, 4]
pic_preg = tf.expand_dims(preg[i], axis=0)
pic_seg = tf.expand_dims(fpn[i], axis=0)
# # [y, x, 1]
# TODO: tensorlite not support squeeze
# category = tf.expand_dims(tf.squeeze(tf.argmax(pic_keypoints, axis=-1, output_type=tf.int32)), axis=-1)
category = tf.expand_dims(tf.argmax(pic_keypoints, axis=-1, output_type=tf.int32)[0], axis=-1)
# [y, x, 1 + 2(y, x) + 1(index_of_class)=4]
meshgrid_xyz = tf.concat([tf.zeros_like(category), tf.cast(center, tf.int32), category], axis=-1)
# [y, x, 1]
pic_keypoints = tf.gather_nd(pic_keypoints, meshgrid_xyz)
# TODO: no necessary to squeeze
# pic_keypoints = tf.squeeze(pic_keypoints)
# [1, y, x, 1(top_value)]
pic_keypoints = tf.expand_dims(pic_keypoints, axis=0)
pic_keypoints = tf.expand_dims(pic_keypoints, axis=-1)
# 3*3 to be peak value
keypoints_peak = self._max_pooling(pic_keypoints, 3, 1)
# mask for each peak_point in each 3*3 area, [1, y, x, 1] (0,1)
keypoints_mask = tf.cast(tf.equal(pic_keypoints, keypoints_peak), tf.float32)
# [1, y, x, 1] (true, false)
pic_keypoints = pic_keypoints * keypoints_mask
# [y*x]
scores = tf.reshape(pic_keypoints, [-1])
# [y*x]
class_id = tf.reshape(category, [-1])
# [(y* x), 2]
grid_yx = tf.reshape(center, [-1, 2])
# [(y*x), 4]
bbox_lrtb = tf.reshape(pic_preg, [-1, 4])
# TODO: manually order and select
# score_mask = scores > self.score_threshold
# scores = tf.boolean_mask(scores, score_mask)
# class_id = tf.boolean_mask(class_id, score_mask)
# grid_yx = tf.boolean_mask(grid_yx, score_mask) + 0.5
# bbox_lrtb = tf.boolean_mask(bbox_lrtb, score_mask)
# TODO: ATTENTION, order are lrtb in prediction, but tlbr in ground_truth
# [num, 4(y1, x1, y2, x2)]
bbox = tf.concat([grid_yx - bbox_lrtb[..., -2::-2], grid_yx + bbox_lrtb[..., -1::-2]], axis=-1)
select_indices = tf.image.non_max_suppression(bbox, scores, self.top_k_results_output,
self.nms_threshold, score_threshold=self.score_threshold)
# [num_select, ?]
select_scores = tf.gather(scores, select_indices)
select_center = tf.gather(grid_yx, select_indices)
select_class_id = tf.gather(class_id, select_indices)
select_bbox = tf.gather(bbox, select_indices)
select_lrtb = tf.gather(bbox_lrtb, select_indices)
class_seg = tf.cast(pic_seg > self.seg_threshold, tf.float32)
# TODO: Could be mute
# final_masks = tf.zeros([pshape[0], pshape[1], tf.shape(select_indices)[0]], tf.float32)
# for i in range(self.num_classes):
# exist_i = tf.equal(select_class_id, i) # [0,1,...]
# exist_int = tf.cast(exist_i, tf.float32)
# index = tf.where(condition=exist_int>0)
# num_i = tf.reduce_sum(exist_int)
# masks = self.seg_instance(index, select_bbox, exist_i, class_seg[0, ..., i], num_i, pic_preg,
# meshgrid_y, meshgrid_x, pshape, tf.shape(select_indices)[0])
# final_masks = final_masks + masks
# end of tensor masks
select_scores = tf.expand_dims(select_scores, axis=0)
select_center = tf.expand_dims(select_center, axis=0)
print("============", tf.shape(select_center))
select_class_id = tf.expand_dims(select_class_id, axis=0)
select_bbox = tf.expand_dims(select_bbox, axis=0)
select_lrtb = tf.expand_dims(select_lrtb, axis=0)
# select_masks = tf.expand_dims(final_masks, axis=0)
# TODO: concatenate the batch
# for post_processing outputs
outputs = [select_center, select_scores, select_bbox, select_class_id, select_lrtb, class_seg, pic_preg]
# # [y, x]
# category = tf.argmax(keypoints[0], axis=2, output_type=tf.int32)
# # [1, y, x, 1]
# max_key = tf.reduce_max(keypoints, axis=2, keepdims=True)
# # 3*3 to be peak value
# peak_key = self._max_pooling(max_key, 3, 1)
# # mask for each peak_point in each 3*3 area, [y, x] (0,1)
# mask_key = tf.cast(tf.equal(max_key, peak_key), tf.float32)
# # [1, y, x, 1]
# mask_key = max_key * mask_key
# # [y*x]
# scores = tf.reshape(mask_key, [-1])
# # [y*x]
# class_id = tf.reshape(category, [-1])
# # [(y* x), 2]
# grid_yx = tf.reshape(center, [-1, 2])
# # [(y*x), 4]
# bbox_lrtb = tf.reshape(preg, [-1, 4])
# # [y*x, 4(y1, x1, y2, x2)]
# bbox = tf.concat([grid_yx - bbox_lrtb[..., -2::-2], grid_yx + bbox_lrtb[..., -1::-2]], axis=-1)
# select_indices = tf.image.non_max_suppression(bbox, scores, self.top_k_results_output,
# self.nms_threshold, score_threshold=self.score_threshold)
# # [num_select, ?]
# select_scores = tf.gather(scores, select_indices)
# select_center = tf.gather(grid_yx, select_indices)
# select_class_id = tf.gather(class_id, select_indices)
# select_bbox = tf.gather(bbox, select_indices)
# # select_lrtb = tf.gather(bbox_lrtb, select_indices)
# class_seg = tf.cast(tf.greater(fpn, self.seg_threshold), tf.float32)
#
# select_scores = tf.expand_dims(select_scores, 0)
# select_bbox = tf.expand_dims(select_bbox, 0)
# select_class_id = tf.expand_dims(select_class_id, 0)
# select_center = tf.expand_dims(select_center, 0)
# print(tf.shape(select_scores))
# print(tf.shape(select_bbox))
# print(tf.shape(select_class_id))
# print(tf.shape(select_center))
# print(tf.shape(class_seg))
outputs = [select_center, select_scores, select_bbox, select_class_id, class_seg, preg]
inputs = [self.images]
self.CenterNetModel = tf.keras.Model(inputs=inputs, outputs=outputs)