def test_serialize_deserialize(self, level, low_level, shared_decoder): """Validate the network can be serialized and deserialized.""" num_classes = 10 backbone = backbones.ResNet(model_id=50) semantic_decoder = aspp.ASPP(level=level, dilation_rates=[6, 12, 18]) if shared_decoder: instance_decoder = semantic_decoder else: instance_decoder = aspp.ASPP(level=level, dilation_rates=[6, 12, 18]) semantic_head = panoptic_deeplab_heads.SemanticHead( num_classes, level=level, low_level=low_level, low_level_num_filters=(64, 32)) instance_head = panoptic_deeplab_heads.InstanceHead( level=level, low_level=low_level, low_level_num_filters=(64, 32)) post_processor = panoptic_deeplab_merge.PostProcessor( output_size=[640, 640], center_score_threshold=0.1, thing_class_ids=[1, 2, 3, 4], label_divisor=[256], stuff_area_limit=4096, ignore_label=0, nms_kernel=41, keep_k_centers=41, rescale_predictions=True) model = panoptic_deeplab_model.PanopticDeeplabModel( backbone=backbone, semantic_decoder=semantic_decoder, instance_decoder=instance_decoder, semantic_head=semantic_head, instance_head=instance_head, post_processor=post_processor) config = model.get_config() new_model = panoptic_deeplab_model.PanopticDeeplabModel.from_config( config) # Validate that the config can be forced to JSON. _ = new_model.to_json() # If the serialization was successful, the new config should match the old. self.assertAllEqual(model.get_config(), new_model.get_config())
def test_serialize_deserialize(self): semantic_head = panoptic_deeplab_heads.SemanticHead(num_classes=2, level=3) instance_head = panoptic_deeplab_heads.InstanceHead(level=3) semantic_head_config = semantic_head.get_config() instance_head_config = instance_head.get_config() new_semantic_head = panoptic_deeplab_heads.SemanticHead.from_config( semantic_head_config) new_instance_head = panoptic_deeplab_heads.InstanceHead.from_config( instance_head_config) self.assertAllEqual(semantic_head.get_config(), new_semantic_head.get_config()) self.assertAllEqual(instance_head.get_config(), new_instance_head.get_config())
def test_forward(self, level, low_level, low_level_num_filters): backbone_features = { '3': np.random.rand(2, 128, 128, 16), '4': np.random.rand(2, 64, 64, 16), '5': np.random.rand(2, 32, 32, 16), } decoder_features = { '3': np.random.rand(2, 128, 128, 64), '4': np.random.rand(2, 64, 64, 64), '5': np.random.rand(2, 32, 32, 64), '6': np.random.rand(2, 16, 16, 64), } backbone_features['2'] = np.random.rand(2, 256, 256, 16) decoder_features['2'] = np.random.rand(2, 256, 256, 64) num_classes = 10 semantic_head = panoptic_deeplab_heads.SemanticHead( num_classes=num_classes, level=level, low_level=low_level, low_level_num_filters=low_level_num_filters) instance_head = panoptic_deeplab_heads.InstanceHead( level=level, low_level=low_level, low_level_num_filters=low_level_num_filters) semantic_outputs = semantic_head((backbone_features, decoder_features)) instance_outputs = instance_head((backbone_features, decoder_features)) if str(level) in decoder_features: h, w = decoder_features[str(low_level[-1])].shape[1:3] self.assertAllEqual(semantic_outputs.numpy().shape, [2, h, w, num_classes]) self.assertAllEqual( instance_outputs['instance_centers_heatmap'].numpy().shape, [2, h, w, 1]) self.assertAllEqual( instance_outputs['instance_centers_offset'].numpy().shape, [2, h, w, 2])
def build_panoptic_deeplab( input_specs: tf.keras.layers.InputSpec, model_config: panoptic_deeplab_cfg.PanopticDeeplab, l2_regularizer: Optional[tf.keras.regularizers.Regularizer] = None ) -> tf.keras.Model: """Builds Panoptic Deeplab model. Args: input_specs: `tf.keras.layers.InputSpec` specs of the input tensor. model_config: Config instance for the panoptic deeplab model. l2_regularizer: Optional `tf.keras.regularizers.Regularizer`, if specified, the model is built with the provided regularization layer. Returns: tf.keras.Model for the panoptic segmentation model. """ norm_activation_config = model_config.norm_activation backbone = backbones.factory.build_backbone( input_specs=input_specs, backbone_config=model_config.backbone, norm_activation_config=norm_activation_config, l2_regularizer=l2_regularizer) semantic_decoder = decoder_factory.build_decoder( input_specs=backbone.output_specs, model_config=model_config, l2_regularizer=l2_regularizer) if model_config.shared_decoder: instance_decoder = None else: # semantic and instance share the same decoder type instance_decoder = decoder_factory.build_decoder( input_specs=backbone.output_specs, model_config=model_config, l2_regularizer=l2_regularizer) semantic_head_config = model_config.semantic_head instance_head_config = model_config.instance_head semantic_head = panoptic_deeplab_heads.SemanticHead( num_classes=model_config.num_classes, level=semantic_head_config.level, num_convs=semantic_head_config.num_convs, kernel_size=semantic_head_config.kernel_size, prediction_kernel_size=semantic_head_config.prediction_kernel_size, num_filters=semantic_head_config.num_filters, use_depthwise_convolution=semantic_head_config. use_depthwise_convolution, upsample_factor=semantic_head_config.upsample_factor, low_level=semantic_head_config.low_level, low_level_num_filters=semantic_head_config.low_level_num_filters, fusion_num_output_filters=semantic_head_config. fusion_num_output_filters, activation=norm_activation_config.activation, use_sync_bn=norm_activation_config.use_sync_bn, norm_momentum=norm_activation_config.norm_momentum, norm_epsilon=norm_activation_config.norm_epsilon, kernel_regularizer=l2_regularizer) instance_head = panoptic_deeplab_heads.InstanceHead( level=instance_head_config.level, num_convs=instance_head_config.num_convs, kernel_size=instance_head_config.kernel_size, prediction_kernel_size=instance_head_config.prediction_kernel_size, num_filters=instance_head_config.num_filters, use_depthwise_convolution=instance_head_config. use_depthwise_convolution, upsample_factor=instance_head_config.upsample_factor, low_level=instance_head_config.low_level, low_level_num_filters=instance_head_config.low_level_num_filters, fusion_num_output_filters=instance_head_config. fusion_num_output_filters, activation=norm_activation_config.activation, use_sync_bn=norm_activation_config.use_sync_bn, norm_momentum=norm_activation_config.norm_momentum, norm_epsilon=norm_activation_config.norm_epsilon, kernel_regularizer=l2_regularizer) if model_config.generate_panoptic_masks: post_processing_config = model_config.post_processor post_processor = panoptic_deeplab_merge.PostProcessor( output_size=post_processing_config.output_size, center_score_threshold=post_processing_config. center_score_threshold, thing_class_ids=post_processing_config.thing_class_ids, label_divisor=post_processing_config.label_divisor, stuff_area_limit=post_processing_config.stuff_area_limit, ignore_label=post_processing_config.ignore_label, nms_kernel=post_processing_config.nms_kernel, keep_k_centers=post_processing_config.keep_k_centers, rescale_predictions=post_processing_config.rescale_predictions) else: post_processor = None model = panoptic_deeplab_model.PanopticDeeplabModel( backbone=backbone, semantic_decoder=semantic_decoder, instance_decoder=instance_decoder, semantic_head=semantic_head, instance_head=instance_head, post_processor=post_processor) return model
def test_panoptic_deeplab_network_creation(self, input_size, level, low_level, shared_decoder, training): """Test for creation of a panoptic deeplab network.""" batch_size = 2 if training else 1 num_classes = 10 inputs = np.random.rand(batch_size, input_size, input_size, 3) image_info = tf.convert_to_tensor([[[input_size, input_size], [input_size, input_size], [1, 1], [0, 0]]]) image_info = tf.tile(image_info, [batch_size, 1, 1]) tf.keras.backend.set_image_data_format('channels_last') backbone = backbones.ResNet(model_id=50) semantic_decoder = aspp.ASPP(level=level, dilation_rates=[6, 12, 18]) if shared_decoder: instance_decoder = semantic_decoder else: instance_decoder = aspp.ASPP(level=level, dilation_rates=[6, 12, 18]) semantic_head = panoptic_deeplab_heads.SemanticHead( num_classes, level=level, low_level=low_level, low_level_num_filters=(64, 32)) instance_head = panoptic_deeplab_heads.InstanceHead( level=level, low_level=low_level, low_level_num_filters=(64, 32)) post_processor = panoptic_deeplab_merge.PostProcessor( output_size=[input_size, input_size], center_score_threshold=0.1, thing_class_ids=[1, 2, 3, 4], label_divisor=[256], stuff_area_limit=4096, ignore_label=0, nms_kernel=41, keep_k_centers=41, rescale_predictions=True) model = panoptic_deeplab_model.PanopticDeeplabModel( backbone=backbone, semantic_decoder=semantic_decoder, instance_decoder=instance_decoder, semantic_head=semantic_head, instance_head=instance_head, post_processor=post_processor) outputs = model(inputs=inputs, image_info=image_info, training=training) if training: self.assertIn('segmentation_outputs', outputs) self.assertIn('instance_centers_heatmap', outputs) self.assertIn('instance_centers_offset', outputs) self.assertAllEqual([ 2, input_size // (2**low_level[-1]), input_size // (2**low_level[-1]), num_classes ], outputs['segmentation_outputs'].numpy().shape) self.assertAllEqual([ 2, input_size // (2**low_level[-1]), input_size // (2**low_level[-1]), 1 ], outputs['instance_centers_heatmap'].numpy().shape) self.assertAllEqual([ 2, input_size // (2**low_level[-1]), input_size // (2**low_level[-1]), 2 ], outputs['instance_centers_offset'].numpy().shape) else: self.assertIn('panoptic_outputs', outputs) self.assertIn('category_mask', outputs) self.assertIn('instance_mask', outputs) self.assertIn('instance_centers', outputs) self.assertIn('instance_scores', outputs) self.assertIn('segmentation_outputs', outputs)