def __init__(self, cfg):
super().__init__()
self.device = torch.device(cfg.MODEL.DEVICE)
self.backbone = build_backbone(cfg)
self.in_features = cfg.MODEL.FCOS.IN_FEATURES
self.num_classes = cfg.MODEL.FCOS.NUM_CLASSES
backbone_shape = self.backbone.output_shape()
feature_shapes = [backbone_shape[f] for f in self.in_features]
#box_selector_train = make_fcos_postprocessor(cfg, is_train=True)
box_selector = make_fcos_postprocessor(cfg) #, is_train=False)
loss_evaluator = make_fcos_loss_evaluator(cfg)
#self.box_selector_train = box_selector_train
self.box_selector = box_selector
self.loss_evaluator = loss_evaluator
self.fpn_strides = cfg.MODEL.FCOS.FPN_STRIDES
self.head = FCOSHead(cfg, feature_shapes[0].channels)
# train on voc or non-voc or all
self.train_part = cfg.MODEL.FCOS.TRAIN_PART
self.feature_strides = {k: v.stride for k, v in backbone_shape.items()}
# mask head
# fmt: off
self.mask_on = cfg.MODEL.MASK_ON
if not self.mask_on:
return
pooler_resolution = cfg.MODEL.ROI_MASK_HEAD.POOLER_RESOLUTION
pooler_scales = tuple(1.0 / self.feature_strides[k] for k in self.in_features)
sampling_ratio = cfg.MODEL.ROI_MASK_HEAD.POOLER_SAMPLING_RATIO
pooler_type = cfg.MODEL.ROI_MASK_HEAD.POOLER_TYPE
# fmt: on
in_channels = feature_shapes[0].channels
self.mask_pooler = ROIPooler(
output_size=pooler_resolution,
scales=pooler_scales,
sampling_ratio=sampling_ratio,
pooler_type=pooler_type,
)
self.mask_head = build_mask_head(
cfg, ShapeSpec(channels=in_channels, width=pooler_resolution, height=pooler_resolution)
)
# Matcher to assign box proposals to gt boxes
self.proposal_matcher = Matcher(
cfg.MODEL.ROI_HEADS.IOU_THRESHOLDS,
cfg.MODEL.ROI_HEADS.IOU_LABELS,
allow_low_quality_matches=False,
)
pixel_mean = torch.Tensor(cfg.MODEL.PIXEL_MEAN).to(self.device).view(3, 1, 1)
pixel_std = torch.Tensor(cfg.MODEL.PIXEL_STD).to(self.device).view(3, 1, 1)
self.normalizer = lambda x: (x - pixel_mean) / pixel_std
self.to(self.device)
def _init_mask_head(cls, cfg, input_shape):
if not cfg.MODEL.MASK_ON:
return {}
# fmt: off
in_features = cfg.MODEL.ROI_HEADS.IN_FEATURES
pooler_resolution = cfg.MODEL.ROI_MASK_HEAD.POOLER_RESOLUTION
pooler_scales = tuple(1.0 / input_shape[k].stride for k in in_features)
sampling_ratio = cfg.MODEL.ROI_MASK_HEAD.POOLER_SAMPLING_RATIO
pooler_type = cfg.MODEL.ROI_MASK_HEAD.POOLER_TYPE
# fmt: on
in_channels = [input_shape[f].channels for f in in_features][0]
ret = {"mask_in_features": in_features}
ret["mask_pooler"] = (ROIPooler(
output_size=pooler_resolution,
scales=pooler_scales,
sampling_ratio=sampling_ratio,
pooler_type=pooler_type,
) if pooler_type else None)
if pooler_type:
shape = ShapeSpec(channels=in_channels,
width=pooler_resolution,
height=pooler_resolution)
else:
shape = {f: input_shape[f] for f in in_features}
ret["mask_head"] = build_mask_head(cfg, shape)
return ret
def _init_mask_head(self, cfg, input_shape):
# fmt: off
self.mask_on = cfg.MODEL.MASK_ON
if not self.mask_on:
return
pooler_resolution = cfg.MODEL.ROI_MASK_HEAD.POOLER_RESOLUTION
pooler_scales = tuple(1.0 / input_shape[k].stride
for k in self.in_features)
sampling_ratio = cfg.MODEL.ROI_MASK_HEAD.POOLER_SAMPLING_RATIO
pooler_type = cfg.MODEL.ROI_MASK_HEAD.POOLER_TYPE
# fmt: on
in_channels = [input_shape[f].channels for f in self.in_features][0]
self.mask_pooler = ROIPooler(
output_size=pooler_resolution,
scales=pooler_scales,
sampling_ratio=sampling_ratio,
pooler_type=pooler_type,
)
self.mask_head = build_mask_head(
cfg,
ShapeSpec(channels=in_channels,
width=pooler_resolution,
height=pooler_resolution))
def __init__(self, cfg, input_shape):
super().__init__(cfg)
# fmt: off
self.in_features = cfg.MODEL.ROI_HEADS.IN_FEATURES
pooler_resolution = cfg.MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION
pooler_type = cfg.MODEL.ROI_BOX_HEAD.POOLER_TYPE
pooler_scales = (1.0 / input_shape[self.in_features[0]].stride, )
sampling_ratio = cfg.MODEL.ROI_BOX_HEAD.POOLER_SAMPLING_RATIO
self.mask_on = cfg.MODEL.MASK_ON
# fmt: on
assert not cfg.MODEL.KEYPOINT_ON
assert len(self.in_features) == 1
self.pooler = ROIPooler(
output_size=pooler_resolution,
scales=pooler_scales,
sampling_ratio=sampling_ratio,
pooler_type=pooler_type,
)
self.res5, out_channels = self._build_res5_block(cfg)
self.box_predictor = FastRCNNOutputLayers(
cfg, ShapeSpec(channels=out_channels, height=1, width=1))
if self.mask_on:
self.mask_head = build_mask_head(
cfg,
ShapeSpec(channels=out_channels,
width=pooler_resolution,
height=pooler_resolution),
)
def _init_mask_head(self, cfg, input_shape):
# fmt: off
self.mask_on = cfg.MODEL.MASK_ON
if not self.mask_on:
return
self.in_features = cfg.MODEL.ROI_HEADS.IN_FEATURES
self.mask_in_features = cfg.MODEL.ROI_HEADS.IN_FEATURES
self._feature_scales = {
k: 1.0 / v.stride
for k, v in input_shape.items()
}
self.pooler_type = cfg.MODEL.ROI_MASK_HEAD.POOLER_TYPE
self.pooler_resolution = cfg.MODEL.ROI_MASK_HEAD.POOLER_RESOLUTION
self.pooler_scales = tuple(1.0 / input_shape[k].stride
for k in self.in_features)
self.sampling_ratio = cfg.MODEL.ROI_MASK_HEAD.POOLER_SAMPLING_RATIO
self.in_channels = [input_shape[f].channels
for f in self.in_features][0]
self.points_num = cfg.MODEL.BORDER_HEAD.POINTS_NUM
if self.pooler_type:
self.mask_pooler = ROIPooler(
output_size=self.pooler_resolution,
scales=self.pooler_scales,
sampling_ratio=self.sampling_ratio,
pooler_type=self.pooler_type,
)
shape = ShapeSpec(channels=self.in_channels,
width=self.pooler_resolution,
height=self.pooler_resolution)
else:
self.mask_pooler = None
shape = {f: input_shape[f] for f in self.in_features}
self.mask_rcnn_head = build_mask_head(cfg, shape)
self.lod_head = build_lod_head(
cfg,
ShapeSpec(
channels=self.in_channels,
width=28,
height=28,
),
)
def _init_mask_head(self, cfg):
# fmt: off
self.mask_on = cfg.MODEL.MASK_ON
if not self.mask_on:
return
self.mask_coarse_in_features = cfg.MODEL.ROI_MASK_HEAD.IN_FEATURES
self.mask_coarse_side_size = cfg.MODEL.ROI_MASK_HEAD.POOLER_RESOLUTION
# fmt: on
in_channels = np.sum(
[self.feature_channels[f] for f in self.mask_coarse_in_features])
self.mask_coarse_head = build_mask_head(
cfg,
ShapeSpec(
channels=in_channels,
width=self.mask_coarse_side_size,
height=self.mask_coarse_side_size,
),
)
self._init_point_head(cfg)
def _init_mask_head(self, cfg, input_shape):
# fmt: off
self.mask_on = cfg.MODEL.MASK_ON
if not self.mask_on:
return
self.mask_coarse_in_features = cfg.MODEL.ROI_MASK_HEAD.IN_FEATURES
self.mask_coarse_side_size = cfg.MODEL.ROI_MASK_HEAD.POOLER_RESOLUTION
self._feature_scales = {k: 1.0 / v.stride for k, v in input_shape.items()}
# fmt: on
in_channels = np.sum([input_shape[f].channels for f in self.mask_coarse_in_features])
self.mask_coarse_head = build_mask_head(
cfg,
ShapeSpec(
channels=in_channels,
width=self.mask_coarse_side_size,
height=self.mask_coarse_side_size,
),
)
self._init_point_head(cfg, input_shape)
def _init_mask_head(self, cfg, input_shape):
self.mask_on = cfg.MODEL.MASK_ON
if not self.mask_on:
return {}
# fmt: off
in_features = cfg.MODEL.ROI_HEADS.IN_FEATURES
pooler_resolution = cfg.MODEL.ROI_MASK_HEAD.POOLER_RESOLUTION
pooler_scales = tuple(1.0 / input_shape[k].stride for k in in_features)
sampling_ratio = cfg.MODEL.ROI_MASK_HEAD.POOLER_SAMPLING_RATIO
pooler_type = cfg.MODEL.ROI_MASK_HEAD.POOLER_TYPE
# edge poolers
context_resolution = cfg.MODEL.CONTEXT_MASK_HEAD.POOLER_RESOLUTION
context_in_features = cfg.MODEL.CONTEXT_MASK_HEAD.IN_FEATURES
context_scales = tuple(1.0 / input_shape[k].stride
for k in context_in_features)
# fmt: on
in_channels = [input_shape[f].channels for f in in_features][0]
self.mask_in_features = in_features
self.context_in_features = context_in_features
# ret = {"mask_in_features": in_features}
self.mask_pooler = ROIPooler(
output_size=pooler_resolution,
scales=pooler_scales,
sampling_ratio=sampling_ratio,
pooler_type=pooler_type,
)
self.context_pooler = ROIPooler(output_size=context_resolution,
scales=context_scales,
sampling_ratio=sampling_ratio,
pooler_type=pooler_type)
self.mask_head = build_mask_head(
cfg,
ShapeSpec(channels=in_channels,
width=pooler_resolution,
height=pooler_resolution))
def __init__(self,
cfg,
d_model,
num_classes,
dim_feedforward=2048,
nhead=8,
dropout=0.1,
activation="relu",
scale_clamp: float = _DEFAULT_SCALE_CLAMP,
bbox_weights=(2.0, 2.0, 1.0, 1.0)):
super().__init__()
self.cfg = cfg
self.d_model = d_model
self.mask_on = cfg.MODEL.MASK_ON
self.num_classes = cfg.MODEL.MSBCNet.NUM_CLASSES
self.train_on_pred_boxes = cfg.MODEL.ROI_BOX_HEAD.TRAIN_ON_PRED_BOXES
self.proposal_append_gt = True
self.test_score_thresh = 0.05
self.test_nms_thresh = 0.5
self.test_topk_per_image = 10
self.score_thresh = 0.05
self.proposal_matcher = Matcher([0.5], [0, 1],
allow_low_quality_matches=False)
# dynamic.
self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
# self.self_attn = MSDeformAttn(d_model, 1, nhead, n_points=4)
self.inst_interact = DynamicConv(cfg)
self.linear1 = nn.Linear(d_model, dim_feedforward)
self.dropout = nn.Dropout(dropout)
self.linear2 = nn.Linear(dim_feedforward, d_model)
self.norm1 = nn.LayerNorm(d_model)
self.norm2 = nn.LayerNorm(d_model)
self.norm3 = nn.LayerNorm(d_model)
self.dropout1 = nn.Dropout(dropout)
self.dropout2 = nn.Dropout(dropout)
self.dropout3 = nn.Dropout(dropout)
self.activation = _get_activation_fn(activation)
# cls.
num_cls = cfg.MODEL.MSBCNet.NUM_CLS
cls_module = list()
for _ in range(num_cls):
cls_module.append(nn.Linear(d_model, d_model, False))
cls_module.append(nn.LayerNorm(d_model))
cls_module.append(nn.ReLU(inplace=True))
self.cls_module = nn.ModuleList(cls_module)
# reg.
num_reg = cfg.MODEL.MSBCNet.NUM_REG
reg_module = list()
for _ in range(num_reg):
reg_module.append(nn.Linear(d_model, d_model, False))
reg_module.append(nn.LayerNorm(d_model))
reg_module.append(nn.ReLU(inplace=True))
self.reg_module = nn.ModuleList(reg_module)
# pred.
self.use_focal = cfg.MODEL.MSBCNet.USE_FOCAL
if self.use_focal:
self.class_logits = nn.Linear(d_model, num_classes)
else:
self.class_logits = nn.Linear(d_model, num_classes + 1)
self.bboxes_delta = nn.Linear(d_model, 4)
self.scale_clamp = scale_clamp
self.bbox_weights = bbox_weights
self.box_pool_resolution = cfg.MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION
self.mask_pool_resolution = cfg.MODEL.ROI_MASK_HEAD.POOLER_RESOLUTION
self.boundary_pool_resolution = cfg.MODEL.BOUNDARY_MASK_HEAD.POOLER_RESOLUTION
self.mask_head = build_mask_head(cfg)
