def inference(self, inputs, weights_path, batch_size=1): self.semantic_segment_model.load_weights(weights_path, by_name=True) self.mask_point_head_model.load_weights(weights_path, by_name=True) # inputs = self.semantic_segment_model.inputs out = self.semantic_segment_model(inputs) mask_logit = out[-1] ResizeShape_list = [128, 256, 320, 480, 512] for subdivision_step, size in enumerate(ResizeShape_list): ResizeShape = (size, size) # ResizeShape = list(map(lambda a: int(a) * 2, mask_logit.shape[1:3])) mask_logit = resize_bilinear(mask_logit, ResizeShape, align_corners=True, name='upsampleing') # R, sH, sW, C = map(int, mask_logit.shape) R, sH, sW, C = mask_logit.shape R = batch_size uncertainty_map = _uncertainty(mask_logit, cls=0) point_indices, point_coords = get_uncertain_point_coords_on_grid( uncertainty_map, # self.num_subdivision_points 5000, batch_size=batch_size) point_coords = point_coords[..., ::-1] coarse_coords = _point_scale2img(point_coords, sH, sW) # local feat # coarse_features = _grid_nd_sample(mask_logit, coarse_coords, batch_dims=1) show_points_image(mask_logit, coarse_coords) fine_gained_feature = [] for mid_output in out: _, fine_H, fine_W, _ = mid_output.shape fine_gained_feature_coords = _point_scale2img( point_coords, fine_H, fine_W) fine_gained_feature.append( _grid_nd_sample(mid_output, fine_gained_feature_coords, batch_dims=1)) fine_gained_feature = tf.concat(fine_gained_feature, axis=-1) point_logits = self.mask_point_head_model(fine_gained_feature) inds = tf.cast(point_coords * tf.constant((sH, sW), tf.float32), tf.int32) expdim = tf.tile( tf.range(0, R, dtype=tf.int32)[..., None], [1, inds.shape[1]])[..., None] inds = tf.concat([expdim, inds], -1) mask_logit = tf.tensor_scatter_nd_update(mask_logit, indices=inds, updates=point_logits) return mask_logit
def coarse_loss(y_true, y_pred): uncertainty_map = _uncertainty(y_pred, cls=0) _, coords = get_uncertain_point_coords_on_grid(uncertainty_map, rate=0.1, batch_size=train_batch_size_per_replica) coords = coords[..., ::-1] _, mid_output_H, mid_output_W, _ = y_pred.shape y_pred_important = grid_nd_sample(y_pred, _point_scale2img(coords, mid_output_H, mid_output_W)) y_true_important = grid_nd_sample(y_true, _point_scale2img(coords, mid_output_H, mid_output_W)) loss1 = tf.keras.losses.binary_crossentropy(y_true_important, y_pred_important) loss2 = tf.keras.losses.binary_crossentropy(y_true, y_pred) loss = tf.concat([20 * tf.reshape(loss1, shape=(train_batch_size_per_replica, -1)), 0.1 * tf.reshape(loss2, shape=(train_batch_size_per_replica, -1))], axis=-1) return loss
def build_model_for_train(self, oversample_ratio=1, important_ratio=0.6): out = self.semantic_segment_model.output coarse = out[-1] # 最后一层的输出为粗分类层 points = get_uncertain_pt_coords_randomness( coarse, batch_size=self.batch_size, oversample_ratio=oversample_ratio, important_ratio=important_ratio, ) fine = [] for mid_output in out: _, mid_output_H, mid_output_W, _ = mid_output.shape fine.append( grid_nd_sample( mid_output, _point_scale2img(points, mid_output_H, mid_output_W))) fine_concated = tf.keras.layers.concatenate(fine, axis=-1) coarse_selected_points = fine[-1] rend = self._mask_point_head(num_classes=self.num_classes, fine_gained_feature=fine_concated) rend_and_coords = tf.keras.layers.concatenate( [coarse_selected_points, rend, points], axis=-1, name='rend_and_coords') coarse = gen_image_ops.resize_bilinear(coarse, size=(512, 512), align_corners=True, name='coarse') # coarse = tf.keras.layers.Lambda(lambda x:gen_image_ops.resize_bilinear(x, size=(512, 512), align_corners=True, ) # ,name='coarse') # coarse = tf.keras.layers.Activation(activation='sigmoid',name='coarse')(coarse) model = tf.keras.Model(inputs=self.semantic_segment_model.inputs, outputs=[coarse, rend_and_coords]) return model
def build_model_for_infer(self, weights_path=None, batch_size=1): self.semantic_segment_model.load_weights(weights_path, by_name=True) self.mask_point_head_model.load_weights(weights_path, by_name=True) inputs = self.semantic_segment_model.inputs out = self.semantic_segment_model.output mask_logit = out[-1] # fine_gained_features = out[:-1] # origin_output = resize_bilinear(mask_logit, (512, 512), align_corners=True, # ) # _, H, W, C = mask_logit.shape ResizeShape_list = [128, 256, 512, 1024, 2048] grid_rate = [0.10, 0.09, 0.08, 0.07, 0.05] for subdivision_step, (size, rate) in enumerate( zip(ResizeShape_list, grid_rate)): ResizeShape = (size, size) # ResizeShape = list(map(lambda a: int(a) * 2, mask_logit.shape[1:3])) mask_logit = resize_bilinear(mask_logit, ResizeShape, align_corners=True, name='upsampleing') # R, sH, sW, C = map(int, mask_logit.shape) R, sH, sW, C = mask_logit.shape R = batch_size uncertainty_map = _uncertainty(mask_logit, cls=0) point_indices, point_coords = get_uncertain_point_coords_on_grid( uncertainty_map, # self.num_subdivision_points rate=rate, batch_size=batch_size) point_coords = point_coords[..., ::-1] # coarse_coords = _point_scale2img(point_coords, sH, sW) # local feat # coarse_features = _grid_nd_sample(mask_logit, coarse_coords, batch_dims=1) # show_points_image(mask_logit, coarse_coords) fine_gained_feature = [] for mid_output in out: _, fine_H, fine_W, _ = mid_output.shape fine_gained_feature_coords = _point_scale2img( point_coords, fine_H, fine_W) fine_gained_feature.append( _grid_nd_sample(mid_output, fine_gained_feature_coords, batch_dims=1)) fine_gained_feature = tf.concat(fine_gained_feature, axis=-1) point_logits = self.mask_point_head_model(fine_gained_feature) inds = tf.cast(point_coords * tf.constant((sH, sW), tf.float32), tf.int32) expdim = tf.tile( tf.range(0, R, dtype=tf.int32)[..., None], [1, inds.shape[1]])[..., None] inds = tf.concat([expdim, inds], -1) mask_logit = tf.tensor_scatter_nd_update(mask_logit, indices=inds, updates=point_logits) model = tf.keras.models.Model(inputs=inputs, outputs=mask_logit) return model
def rend_and_coords_acc(self, y_true, y_pred): y_true = tf.expand_dims(y_true, axis=-1) _, H, W, _ = y_true.shape coords = y_pred[..., -2:] coords = _point_scale2img(coords, H, W) rend = y_pred[..., -2 - self.num_classes:-2] y_true_ = _grid_nd_sample(in_tensor=y_true, indices=coords, batch_dims=1) return tf.keras.metrics.binary_accuracy(y_true=y_true_, y_pred=rend)
def pointrend_loss(self, y_true, y_pred): y_true = tf.expand_dims(y_true, axis=-1) _, H, W, _ = y_true.shape coords = y_pred[..., -2:] coords = _point_scale2img(coords, H, W) rend = y_pred[..., -2 - self.num_classes:-2] y_true_ = _grid_nd_sample(in_tensor=y_true, indices=coords, batch_dims=1) # # num_positive1 = tf.reduce_sum(y_true) # num_negtive1 = tf.reduce_sum(1 - y_true) # tf.print(num_positive1, num_negtive1) # num_positive = tf.reduce_sum(y_true_) # num_negtive = tf.reduce_sum(1 - y_true_) # tf.print(num_positive, num_negtive) return tf.keras.losses.binary_crossentropy(y_true=y_true_, y_pred=rend)