def inference(self, inputs, weights_path, batch_size=1):
self.semantic_segment_model.load_weights(weights_path, by_name=True)
self.mask_point_head_model.load_weights(weights_path, by_name=True)
# inputs = self.semantic_segment_model.inputs
out = self.semantic_segment_model(inputs)
mask_logit = out[-1]
ResizeShape_list = [128, 256, 320, 480, 512]
for subdivision_step, size in enumerate(ResizeShape_list):
ResizeShape = (size, size)
# ResizeShape = list(map(lambda a: int(a) * 2, mask_logit.shape[1:3]))
mask_logit = resize_bilinear(mask_logit,
ResizeShape,
align_corners=True,
name='upsampleing')
# R, sH, sW, C = map(int, mask_logit.shape)
R, sH, sW, C = mask_logit.shape
R = batch_size
uncertainty_map = _uncertainty(mask_logit, cls=0)
point_indices, point_coords = get_uncertain_point_coords_on_grid(
uncertainty_map,
# self.num_subdivision_points
5000,
batch_size=batch_size)
point_coords = point_coords[..., ::-1]
coarse_coords = _point_scale2img(point_coords, sH,
sW) # local feat
# coarse_features = _grid_nd_sample(mask_logit, coarse_coords, batch_dims=1)
show_points_image(mask_logit, coarse_coords)
fine_gained_feature = []
for mid_output in out:
_, fine_H, fine_W, _ = mid_output.shape
fine_gained_feature_coords = _point_scale2img(
point_coords, fine_H, fine_W)
fine_gained_feature.append(
_grid_nd_sample(mid_output,
fine_gained_feature_coords,
batch_dims=1))
fine_gained_feature = tf.concat(fine_gained_feature, axis=-1)
point_logits = self.mask_point_head_model(fine_gained_feature)
inds = tf.cast(point_coords * tf.constant((sH, sW), tf.float32),
tf.int32)
expdim = tf.tile(
tf.range(0, R, dtype=tf.int32)[..., None],
[1, inds.shape[1]])[..., None]
inds = tf.concat([expdim, inds], -1)
mask_logit = tf.tensor_scatter_nd_update(mask_logit,
indices=inds,
updates=point_logits)
return mask_logit
def coarse_loss(y_true, y_pred):
uncertainty_map = _uncertainty(y_pred, cls=0)
_, coords = get_uncertain_point_coords_on_grid(uncertainty_map, rate=0.1,
batch_size=train_batch_size_per_replica)
coords = coords[..., ::-1]
_, mid_output_H, mid_output_W, _ = y_pred.shape
y_pred_important = grid_nd_sample(y_pred, _point_scale2img(coords, mid_output_H, mid_output_W))
y_true_important = grid_nd_sample(y_true, _point_scale2img(coords, mid_output_H, mid_output_W))
loss1 = tf.keras.losses.binary_crossentropy(y_true_important, y_pred_important)
loss2 = tf.keras.losses.binary_crossentropy(y_true, y_pred)
loss = tf.concat([20 * tf.reshape(loss1, shape=(train_batch_size_per_replica, -1)),
0.1 * tf.reshape(loss2, shape=(train_batch_size_per_replica, -1))], axis=-1)
return loss
def build_model_for_train(self, oversample_ratio=1, important_ratio=0.6):
out = self.semantic_segment_model.output
coarse = out[-1] # 最后一层的输出为粗分类层
points = get_uncertain_pt_coords_randomness(
coarse,
batch_size=self.batch_size,
oversample_ratio=oversample_ratio,
important_ratio=important_ratio,
)
fine = []
for mid_output in out:
_, mid_output_H, mid_output_W, _ = mid_output.shape
fine.append(
grid_nd_sample(
mid_output,
_point_scale2img(points, mid_output_H, mid_output_W)))
fine_concated = tf.keras.layers.concatenate(fine, axis=-1)
coarse_selected_points = fine[-1]
rend = self._mask_point_head(num_classes=self.num_classes,
fine_gained_feature=fine_concated)
rend_and_coords = tf.keras.layers.concatenate(
[coarse_selected_points, rend, points],
axis=-1,
name='rend_and_coords')
coarse = gen_image_ops.resize_bilinear(coarse,
size=(512, 512),
align_corners=True,
name='coarse')
# coarse = tf.keras.layers.Lambda(lambda x:gen_image_ops.resize_bilinear(x, size=(512, 512), align_corners=True, )
# ,name='coarse')
# coarse = tf.keras.layers.Activation(activation='sigmoid',name='coarse')(coarse)
model = tf.keras.Model(inputs=self.semantic_segment_model.inputs,
outputs=[coarse, rend_and_coords])
return model
def build_model_for_infer(self, weights_path=None, batch_size=1):
self.semantic_segment_model.load_weights(weights_path, by_name=True)
self.mask_point_head_model.load_weights(weights_path, by_name=True)
inputs = self.semantic_segment_model.inputs
out = self.semantic_segment_model.output
mask_logit = out[-1]
# fine_gained_features = out[:-1]
# origin_output = resize_bilinear(mask_logit, (512, 512), align_corners=True,
# )
# _, H, W, C = mask_logit.shape
ResizeShape_list = [128, 256, 512, 1024, 2048]
grid_rate = [0.10, 0.09, 0.08, 0.07, 0.05]
for subdivision_step, (size, rate) in enumerate(
zip(ResizeShape_list, grid_rate)):
ResizeShape = (size, size)
# ResizeShape = list(map(lambda a: int(a) * 2, mask_logit.shape[1:3]))
mask_logit = resize_bilinear(mask_logit,
ResizeShape,
align_corners=True,
name='upsampleing')
# R, sH, sW, C = map(int, mask_logit.shape)
R, sH, sW, C = mask_logit.shape
R = batch_size
uncertainty_map = _uncertainty(mask_logit, cls=0)
point_indices, point_coords = get_uncertain_point_coords_on_grid(
uncertainty_map,
# self.num_subdivision_points
rate=rate,
batch_size=batch_size)
point_coords = point_coords[..., ::-1]
# coarse_coords = _point_scale2img(point_coords, sH, sW) # local feat
# coarse_features = _grid_nd_sample(mask_logit, coarse_coords, batch_dims=1)
# show_points_image(mask_logit, coarse_coords)
fine_gained_feature = []
for mid_output in out:
_, fine_H, fine_W, _ = mid_output.shape
fine_gained_feature_coords = _point_scale2img(
point_coords, fine_H, fine_W)
fine_gained_feature.append(
_grid_nd_sample(mid_output,
fine_gained_feature_coords,
batch_dims=1))
fine_gained_feature = tf.concat(fine_gained_feature, axis=-1)
point_logits = self.mask_point_head_model(fine_gained_feature)
inds = tf.cast(point_coords * tf.constant((sH, sW), tf.float32),
tf.int32)
expdim = tf.tile(
tf.range(0, R, dtype=tf.int32)[..., None],
[1, inds.shape[1]])[..., None]
inds = tf.concat([expdim, inds], -1)
mask_logit = tf.tensor_scatter_nd_update(mask_logit,
indices=inds,
updates=point_logits)
model = tf.keras.models.Model(inputs=inputs, outputs=mask_logit)
return model
def rend_and_coords_acc(self, y_true, y_pred):
y_true = tf.expand_dims(y_true, axis=-1)
_, H, W, _ = y_true.shape
coords = y_pred[..., -2:]
coords = _point_scale2img(coords, H, W)
rend = y_pred[..., -2 - self.num_classes:-2]
y_true_ = _grid_nd_sample(in_tensor=y_true,
indices=coords,
batch_dims=1)
return tf.keras.metrics.binary_accuracy(y_true=y_true_, y_pred=rend)
def pointrend_loss(self, y_true, y_pred):
y_true = tf.expand_dims(y_true, axis=-1)
_, H, W, _ = y_true.shape
coords = y_pred[..., -2:]
coords = _point_scale2img(coords, H, W)
rend = y_pred[..., -2 - self.num_classes:-2]
y_true_ = _grid_nd_sample(in_tensor=y_true,
indices=coords,
batch_dims=1)
#
# num_positive1 = tf.reduce_sum(y_true)
# num_negtive1 = tf.reduce_sum(1 - y_true)
# tf.print(num_positive1, num_negtive1)
# num_positive = tf.reduce_sum(y_true_)
# num_negtive = tf.reduce_sum(1 - y_true_)
# tf.print(num_positive, num_negtive)
return tf.keras.losses.binary_crossentropy(y_true=y_true_, y_pred=rend)