def _test_application_notop(name, input_shape=(None, None, 3), last_dim=1024): classifier = Classifiers.get_classifier(name) model = classifier(input_shape=input_shape, weights=None, include_top=False) assert model.output_shape == (None, None, None, last_dim)
def get_backbone(name, *args, **kwargs): return Classifiers.get_classifier(name)(*args, **kwargs)
def build_maskrcnn(config): # Image size must be dividable by 2 multiple times if config.IMAGE_WIDTH % 64 != 0 and config.IMAGE_HEIGHT % 64 != 0: raise Exception('Image size must be dividable by 2 at least 6 times ' 'to avoid fractions when downscaling and upscaling.' 'For example, use 256, 320, 384, 448, 512, ... etc. ') image_shape = config.IMAGE_SHAPE # Inputs input_image = Input( shape=image_shape, name='input_image') input_image_meta = Input(shape=[config.IMAGE_META_SIZE], name='input_image_meta') classifier = Classifiers.get_classifier(config.BACKBONE) backbone = classifier(input_tensor=input_image, input_shape=image_shape, include_top=False, weights=config.BACKBONE_WEIGHTS) backbone_layer_names = [] for layer in backbone.layers: backbone_layer_names.append(layer.name) feature_layers = get_maskrcnn_feature_layers(config, backbone) # convert layer names to indices skip_connection_idx = ([get_layer_number(backbone, l) if isinstance(l, str) else l for l in feature_layers]) backbone_features = [] for idx in skip_connection_idx: backbone_features.append(backbone.layers[idx].output) feature_pyramids = fpn_graph(backbone_features, config.TOP_DOWN_PYRAMID_SIZE) rpn_feature_maps = feature_pyramids mrcnn_feature_maps = feature_pyramids[:-1] # Anchors if config.MODE == 'training': anchors = get_anchors(config) # Duplicate across the batch dimension because Keras requires it # TODO: can this be optimized to avoid duplicating the anchors? anchors = np.broadcast_to(anchors, (config.BATCH_SIZE,) + anchors.shape) # A hack to get around Keras's bad support for constants anchors = Lambda(lambda x: tf.Variable(anchors), name='anchors')(input_image) else: # Anchors in normalized coordinates anchors = Input(shape=[None, 4], name='input_anchors') # RPN Model rpn = RegionProposalNet(config.RPN_ANCHOR_STRIDE, len(config.RPN_ANCHOR_RATIOS), config.TOP_DOWN_PYRAMID_SIZE) # Loop through FPN outputs layer_outputs = [] # list of lists for p in rpn_feature_maps: layer_outputs.append(rpn([p])) # Concatenate layer outputs # Convert from list of lists of level outputs to list of lists # of outputs across levels. # e.g. [[a1, b1, c1], [a2, b2, c2]] => [[a1, a2], [b1, b2], [c1, c2]] output_names = ['rpn_class_logits', 'rpn_class', 'rpn_bbox'] outputs = list(zip(*layer_outputs)) outputs = [Concatenate(axis=1, name=n)(list(o)) for o, n in zip(outputs, output_names)] ''' input: (B, N, N, 3) anchors_per_location: len(config.RPN_ANCHOR_RATIOS) the coef: 341 / 4096 = (1/4)**2 + (1/8)**2 + (1/16)**2 + (1/32)**2 + (1/64)**2 rpn_class_logits: (B, anchors_per_location * N * N * (341 / 4096), 2) rpn_class: (B, anchors_per_location * N * N * (341 / 4096), 2) rpn_bbox: (B, anchors_per_location * N * N * (341 / 4096), 4) ''' rpn_class_logits, rpn_class, rpn_bbox = outputs # Generate proposals # Proposals are [batch, N, (y1, x1, y2, x2)] in normalized coordinates # and zero padded. proposal_count = config.POST_NMS_ROIS_TRAINING if config.MODE == 'training' \ else config.POST_NMS_ROIS_INFERENCE rpn_rois = ProposalLayer( proposal_count=proposal_count, nms_threshold=config.RPN_NMS_THRESHOLD, name='ROI', config=config)([rpn_class, rpn_bbox, anchors]) if config.MODE == 'training': # Create all the input layers # RPN GT input_rpn_match = Input( shape=[None, 1], name='input_rpn_match', dtype=tf.int32) input_rpn_bbox = Input( shape=[None, 4], name='input_rpn_bbox', dtype=tf.float32) # Detection GT (class IDs, bounding boxes, and masks) # 1. GT Class IDs (zero padded) input_gt_class_ids = Input( shape=[None], name='input_gt_class_ids', dtype=tf.int32) # 2. GT Boxes in pixels (zero padded) # [batch, MAX_GT_INSTANCES, (y1, x1, y2, x2)] in image coordinates input_gt_boxes = Input( shape=[None, 4], name='input_gt_boxes', dtype=tf.float32) # Normalize coordinates gt_boxes = Lambda(lambda x: norm_boxes_graph( x, shape(input_image)[1:3]))(input_gt_boxes) # 3. GT Masks (zero padded) # [batch, height, width, MAX_GT_INSTANCES] if config.USE_MINI_MASK: input_gt_masks = Input( shape=[config.MINI_MASK_SHAPE[0], config.MINI_MASK_SHAPE[1], None], name='input_gt_masks', dtype=bool) else: input_gt_masks = Input( shape=[image_shape[0], image_shape[1], None], name='input_gt_masks', dtype=bool) # Class ID mask to mark class IDs supported by the dataset the image # came from. active_class_ids = Lambda( lambda x: parse_image_meta_graph(x)['active_class_ids'] )(input_image_meta) # Generate detection targets # Subsamples proposals and generates target outputs for training # Note that proposal class IDs, gt_boxes, and gt_masks are zero # padded. Equally, returned rois and targets are zero padded. rois, target_class_ids, target_bbox, target_mask = \ DetectionTargetLayer(config, name='proposal_targets')([ rpn_rois, input_gt_class_ids, gt_boxes, input_gt_masks]) # Network Heads # TODO: verify that this handles zero padded ROIs mrcnn_class_logits, mrcnn_class, mrcnn_bbox = \ mrcnn_classifier_graph(rois, mrcnn_feature_maps, input_image_meta, config.POOL_SIZE, config.NUM_CLASSES, fc_layers_size=config.FPN_CLASSIF_FC_LAYERS_SIZE) mrcnn_mask = mrcnn_mask_graph(rois, mrcnn_feature_maps, input_image_meta, config.MASK_POOL_SIZE, config.NUM_CLASSES) # TODO: clean up (use tf.identify if necessary) output_rois = Lambda(lambda x: x * 1, name='output_rois')(rois) # Losses rpn_class_loss = Lambda(lambda x: rpn_class_loss_graph(*x), name='rpn_class_loss')( [input_rpn_match, rpn_class_logits]) rpn_bbox_loss = Lambda(lambda x: rpn_bbox_loss_graph(config, *x), name='rpn_bbox_loss')( [input_rpn_bbox, input_rpn_match, rpn_bbox]) class_loss = Lambda(lambda x: mrcnn_class_loss_graph(*x), name='mrcnn_class_loss')( [target_class_ids, mrcnn_class_logits, active_class_ids]) bbox_loss = Lambda(lambda x: mrcnn_bbox_loss_graph(*x), name='mrcnn_bbox_loss')( [target_bbox, target_class_ids, mrcnn_bbox]) mask_loss = Lambda(lambda x: mrcnn_mask_loss_graph(*x), name='mrcnn_mask_loss')( [target_mask, target_class_ids, mrcnn_mask]) # Model inputs = [input_image, input_image_meta, input_rpn_match, input_rpn_bbox, input_gt_class_ids, input_gt_boxes, input_gt_masks] outputs = [rpn_class_logits, rpn_class, rpn_bbox, mrcnn_class_logits, mrcnn_class, mrcnn_bbox, mrcnn_mask, rpn_rois, output_rois, rpn_class_loss, rpn_bbox_loss, class_loss, bbox_loss, mask_loss] model = Model(inputs, outputs, name='maskrcnn') else: # Network Heads # Proposal classifier and BBox regressor heads mrcnn_class_logits, mrcnn_class, mrcnn_bbox = \ mrcnn_classifier_graph(rpn_rois, mrcnn_feature_maps, input_image_meta, config.POOL_SIZE, config.NUM_CLASSES, fc_layers_size=config.FPN_CLASSIF_FC_LAYERS_SIZE) # Detections # output is [batch, num_detections, (y1, x1, y2, x2, class_id, score)] in # normalized coordinates detections = DetectionLayer(config, name='mrcnn_detection')( [rpn_rois, mrcnn_class, mrcnn_bbox, input_image_meta]) # Create masks for detections detection_boxes = Lambda(lambda x: x[..., :4])(detections) mrcnn_mask = mrcnn_mask_graph(detection_boxes, mrcnn_feature_maps, input_image_meta, config.MASK_POOL_SIZE, config.NUM_CLASSES) model = Model([input_image, input_image_meta, anchors], [detections, mrcnn_class, mrcnn_bbox, mrcnn_mask, rpn_rois, rpn_class, rpn_bbox], name='maskrcnn') return model, backbone_layer_names