def build_resnet_fpn_backbone(cfg):
body = resnet.ResNet(cfg)
in_channels_stage2 = cfg.MODEL.RESNETS.RES2_OUT_CHANNELS
out_channels = cfg.MODEL.RESNETS.BACKBONE_OUT_CHANNELS
fpn = fpn_module.FPN(
in_channels_list=[
in_channels_stage2,
in_channels_stage2 * 2,
in_channels_stage2 * 4,
in_channels_stage2 * 8,
],
out_channels=out_channels,
conv_block=conv_with_kaiming_uniform(cfg.MODEL.FPN.USE_GN, cfg.MODEL.FPN.USE_RELU),
top_blocks=fpn_module.LastLevelMaxPool(),
)
model = nn.Sequential(OrderedDict([("body", body), ("fpn", fpn)]))
model.out_channels = out_channels
return model
def build_mnv2_fpn_backbone(cfg):
body = mobilenet.MobileNetV2(cfg)
in_channels_stage2 = body.return_features_num_channels
out_channels = cfg.MODEL.RESNETS.BACKBONE_OUT_CHANNELS
fpn = fpn_module.FPN(
in_channels_list=[
0,
in_channels_stage2[1],
in_channels_stage2[2],
in_channels_stage2[3],
],
out_channels=out_channels,
conv_block=conv_with_kaiming_uniform(cfg.MODEL.FPN.USE_GN,
cfg.MODEL.FPN.USE_RELU),
top_blocks=fpn_module.LastLevelP6P7(out_channels, out_channels),
)
model = nn.Sequential(OrderedDict([("body", body), ("fpn", fpn)]))
model.out_channels = out_channels
return model
def build_mobilenet_fpn_backbone(cfg):
body = mobilenet.MobileNetV2(cfg)
out_channels = cfg.MODEL.MOBILENET.OUT_CHANNELS
fpn = fpn_module.FPN(
cfg,
in_channels_list=[
0,
32,
96,
cfg.MODEL.MOBILENET.LAST_CHANNELS,
],
out_channels=out_channels,
conv_block=conv_with_kaiming_uniform(cfg.MODEL.FPN.USE_GN,
cfg.MODEL.FPN.USE_RELU),
top_blocks=None,
)
model = nn.Sequential(OrderedDict([("body", body), ("fpn", fpn)]))
model.out_channels = out_channels
return model
def __init__(self,
in_channels,
num_levels,
refine_level=1,
refine_type=None,
use_gn=False,
use_deformable=False):
"""
Arguments:
in_channels (int): Number of input channels (feature maps of all levels
should have the same channels).
num_levels (int): Number of input feature levels.
refine_level (int): Index of integration and refine level of BSF in
multi-level features from bottom to top.
refine_type (str): Type of the refine op, currently support
[None, 'conv', 'non_local'].
"""
super(BFP, self).__init__()
assert refine_type in [None, 'conv', 'non_local', 'gc_block']
self.in_channels = in_channels
self.num_levels = num_levels
self.refine_level = refine_level
self.refine_type = refine_type
assert 0 <= self.refine_level < self.num_levels
if self.refine_type == 'conv':
conv_block = conv_with_kaiming_uniform(
use_gn=use_gn,
use_deformable=use_deformable)
self.refine = conv_block(
self.in_channels,
self.in_channels,
3,
padding=1,
conv_cfg=self.conv_cfg,
norm_cfg=self.norm_cfg)
elif self.refine_type == 'gc_block':
self.refine = ContextBlock(
self.in_channels,
ratio=1. / 16.)
def build_resnet_fpn_p3p7_backbone(cfg):
body = resnet.ResNet(cfg)
in_channels_stage2 = cfg.MODEL.RESNETS.RES2_OUT_CHANNELS
out_channels = cfg.MODEL.BACKBONE.OUT_CHANNELS
in_channels_p6p7 = in_channels_stage2 * 8 if cfg.MODEL.RETINANET.USE_C5 \
else out_channels
fpn = fpn_module.FPN(
in_channels_list=[
0,
in_channels_stage2 * 2,
in_channels_stage2 * 4,
in_channels_stage2 * 8,
],
out_channels=out_channels,
conv_block=conv_with_kaiming_uniform(cfg.MODEL.FPN.USE_GN,
cfg.MODEL.FPN.USE_RELU),
top_blocks=fpn_module.LastLevelP6P7(in_channels_p6p7, out_channels),
)
model = nn.Sequential(OrderedDict([("body", body), ("fpn", fpn)]))
return model
def build_detnet_fpn_backbone(cfg):
body = detnet.DetNet(cfg)
in_channels_stage2 = cfg.MODEL.RESNETS.RES2_OUT_CHANNELS
out_channels = cfg.MODEL.RESNETS.BACKBONE_OUT_CHANNELS
fpn = fpn_module.FPN_detnet(
in_channels_list=[
in_channels_stage2,
in_channels_stage2 * 2,
in_channels_stage2 * 4,
in_channels_stage2 * 4,
in_channels_stage2 * 4,
],
out_channels=out_channels,
conv_block=conv_with_kaiming_uniform(cfg.MODEL.FPN.NORM_FUNC,
cfg.MODEL.FPN.USE_RELU),
top_blocks=None,
)
model = nn.Sequential(OrderedDict([("body", body), ("fpn", fpn)]))
model.out_channels = out_channels
return model
def build_prob_depth_estimation_resnet_backbone(cfg):
body = depth_resnet.resnet50(pretrained=False) # default depth prob = 0.5
body.depth_prob = cfg.MODEL.BACKBONE.DEPTH_PROB
in_channels_stage2 = cfg.MODEL.RESNETS.RES2_OUT_CHANNELS
out_channels = cfg.MODEL.RESNETS.BACKBONE_OUT_CHANNELS
fpn = fpn_module.FPN(
in_channels_list=[
in_channels_stage2,
in_channels_stage2 * 2,
in_channels_stage2 * 4,
in_channels_stage2 * 8,
],
out_channels=out_channels,
conv_block=conv_with_kaiming_uniform(cfg.MODEL.FPN.USE_GN,
cfg.MODEL.FPN.USE_RELU),
top_blocks=fpn_module.LastLevelMaxPool(),
)
model = nn.Sequential(OrderedDict([("body", body), ("fpn", fpn)]))
model.out_channels = out_channels
return model
def build_detnasnet_fpn_backbone(cfg):
in_channels_stage2 = cfg.MODEL.HNASNET.FILTER_MULTIPLIER
in_channels_list = [
in_channels_stage2 * s for s in cfg.MODEL.HNASNET.STRIDE_MULTIPLIER[1:]
]
in_channels_list = [0] + in_channels_list
body = DetNASNet(cfg)
out_channels = cfg.MODEL.HNASNET.BACKBONE_OUT_CHANNELS
fpn = fpn_module.Scaler(
in_channels_list=in_channels_list,
out_channels=out_channels,
conv_block=conv_with_kaiming_uniform(cfg.MODEL.FPN.USE_GN,
cfg.MODEL.FPN.USE_RELU,
cfg.MODEL.FPN.USE_DEFORMABLE,
cfg.MODEL.FPN.USE_BN),
top_blocks=fpn_module.LastLevelMaxPool(),
)
model = nn.Sequential(OrderedDict([("body", body), ("fpn", fpn)]))
model.out_channels = out_channels
return model
def build_vovnet_fpn_backbone(cfg):
body = vovnet.VoVNet(cfg)
in_channels_stage = cfg.MODEL.VOVNET.OUT_CHANNELS
out_channels = cfg.MODEL.VOVNET.BACKBONE_OUT_CHANNELS
in_channels_p6p7 = in_channels_stage * 4 if cfg.MODEL.RETINANET.USE_C5 \
else out_channels
fpn = fpn_module.FPN(
in_channels_list=[
0,
in_channels_stage * 2,
in_channels_stage * 3,
in_channels_stage * 4,
],
out_channels=out_channels,
conv_block=conv_with_kaiming_uniform(cfg.MODEL.FPN.USE_GN,
cfg.MODEL.FPN.USE_RELU),
top_blocks=fpn_module.LastLevelP6P7(in_channels_p6p7, out_channels),
)
model = nn.Sequential(OrderedDict([("body", body), ("fpn", fpn)]))
model.out_channels = out_channels
return model
def build_depth_resnet_backbone(cfg):
if cfg.MODEL.RESNETS.PRETRAINED:
body = depth_resnet_pretrained.resnet50(
pretrained=cfg.MODEL.RESNETS.PRETRAINED)
#body = depth_resnet.resnet50(pretrained=cfg.MODEL.RESNETS.PRETRAINED)
weights_rgb = body.conv1.weight.data
depth_conv1 = nn.Conv2d(4,
64,
kernel_size=(7, 7),
stride=(2, 2),
padding=(3, 3),
bias=False)
depth_conv1.weight.requires_grad = False
depth_conv1.weight[:, :3] = weights_rgb
depth_conv1.weight.requires_grad = True
body.conv1 = depth_conv1
else:
body = depth_resnet.resnet50(pretrained=cfg.MODEL.RESNETS.PRETRAINED)
in_channels_stage2 = cfg.MODEL.RESNETS.RES2_OUT_CHANNELS
out_channels = cfg.MODEL.RESNETS.BACKBONE_OUT_CHANNELS
fpn = fpn_module.FPN(
in_channels_list=[
in_channels_stage2,
in_channels_stage2 * 2,
in_channels_stage2 * 4,
in_channels_stage2 * 8,
],
out_channels=out_channels,
conv_block=conv_with_kaiming_uniform(cfg.MODEL.FPN.USE_GN,
cfg.MODEL.FPN.USE_RELU),
top_blocks=fpn_module.LastLevelMaxPool(),
)
model = nn.Sequential(OrderedDict([("body", body), ("fpn", fpn)]))
model.out_channels = out_channels
return model
def build_mnv2_backbone(cfg):
body = mobilenet.MobileNetV2(cfg)
in_channels_stage2 = cfg.MODEL.MOBILENET.OUT_CHANNELS
out_channels = cfg.MODEL.BACKBONE.OUT_CHANNELS
in_channels_p6p7 = in_channels_stage2 * 8 if cfg.MODEL.RETINANET.USE_C5 \
else out_channels
fpn = fpn_module.FPN(
in_channels_list=[
0,
in_channels_stage2[1],
in_channels_stage2[2],
in_channels_stage2[3],
],
out_channels=out_channels,
conv_block=conv_with_kaiming_uniform(cfg.MODEL.FPN.USE_GN,
cfg.MODEL.FPN.USE_RELU),
top_blocks=fpn_module.LastLevelP6P7(in_channels_p6p7, out_channels),
)
model = nn.Sequential(OrderedDict([("body", body), ("fpn", fpn)]))
model.out_channels = out_channels
return model
def build_resnet_fpn_backbone(cfg, fuse_factors):
body = resnet.ResNet(cfg)
in_channels_stage2 = cfg.MODEL.RESNETS.RES2_OUT_CHANNELS
out_channels = cfg.MODEL.RESNETS.BACKBONE_OUT_CHANNELS
fpn = fpn_module.FPN(
in_channels_list=[
in_channels_stage2,
in_channels_stage2 * 2,
in_channels_stage2 * 4,
in_channels_stage2 * 8,
],
out_channels=out_channels,
conv_block=conv_with_kaiming_uniform(cfg.MODEL.FPN.USE_GN,
cfg.MODEL.FPN.USE_RELU),
top_blocks=fpn_module.LastLevelMaxPool(),
upsample_rates=cfg.MODEL.FPN.UPSAMPLE_RATE, # add by hui
upsample_mode=cfg.MODEL.FPN.UPSAMPLE_MODE, # add by hui
fusion_factors=fuse_factors # add by G
)
model = nn.Sequential(OrderedDict([("body", body), ("fpn", fpn)]))
model.out_channels = out_channels
return model
def build_baseline_resnet_backbone(cfg):
body = baseline_resnet.resnet50(pretrained=cfg.MODEL.RESNETS.PRETRAINED)
in_channels_stage2 = cfg.MODEL.RESNETS.RES2_OUT_CHANNELS
out_channels = cfg.MODEL.RESNETS.BACKBONE_OUT_CHANNELS
fpn = fpn_module.FPN(
in_channels_list=[
in_channels_stage2,
in_channels_stage2 * 2,
in_channels_stage2 * 4,
in_channels_stage2 * 8,
],
out_channels=out_channels,
conv_block=conv_with_kaiming_uniform(cfg.MODEL.FPN.USE_GN,
cfg.MODEL.FPN.USE_RELU),
top_blocks=fpn_module.LastLevelMaxPool(),
)
# append resent with the feature pyramid network
model = nn.Sequential(OrderedDict([("body", body), ("fpn", fpn)]))
model.out_channels = out_channels
return model
def build_shufflenet_fpn_backbone(cfg):
from qd.qd_common import load_from_yaml_str
param = cfg.MODEL.BACKBONE.CONV_BODY_PARAM
param = load_from_yaml_str(param)
from qd.qd_common import execute_func
body = execute_func(param)
out_channels = cfg.MODEL.RESNETS.BACKBONE_OUT_CHANNELS
fpn = fpn_module.FPN(
in_channels_list=body.get_out_channels(),
out_channels=out_channels,
conv_block=conv_with_kaiming_uniform(
cfg.MODEL.FPN.USE_BN,
cfg.MODEL.FPN.USE_GN,
cfg.MODEL.FPN.USE_RELU
),
top_blocks=fpn_module.LastLevelMaxPool(),
interpolate_mode=cfg.MODEL.FPN.INTERPOLATE_MODE,
)
model = nn.Sequential(OrderedDict([("body", body), ("fpn", fpn)]))
model.out_channels = out_channels
return model
def build_hnasnet_backbone(cfg):
in_channels_stage2 = (cfg.MODEL.HNASNET.FILTER_MULTIPLIER *
cfg.MODEL.HNASNET.NUM_BLOCKS)
in_channels_list = [
in_channels_stage2 * 2**i
for i in range(1, len(cfg.MODEL.HNASNET.STRIDE_MULTIPLIER))
]
in_channels_list = [0] + in_channels_list
body = HNASNet(cfg)
out_channels = cfg.MODEL.HNASNET.BACKBONE_OUT_CHANNELS
in_channels_p6p7 = in_channels_stage2 * 8 if cfg.MODEL.RETINANET.USE_C5 \
else out_channels
fpn = fpn_module.FPN(
in_channels_list=in_channels_list,
out_channels=out_channels,
conv_block=conv_with_kaiming_uniform(cfg.MODEL.FPN.USE_GN,
cfg.MODEL.FPN.USE_RELU,
cfg.MODEL.FPN.USE_DEFORMABLE),
top_blocks=fpn_module.LastLevelP6P7(in_channels_p6p7, out_channels),
)
model = nn.Sequential(OrderedDict([("body", body), ("fpn", fpn)]))
model.out_channels = out_channels
return model
def build_resnet_fpn_p3p7_backbone(stem_out_channels=64,
in_channels_stage2=256,
out_channels=256,
use_c5=False,
use_gn=False,
use_relu=False,
num_input_channels=4):
stem_module = resnet.StemWithFixedBatchNorm(stem_out_channels,
in_channels=num_input_channels)
stage_specs = [ # R-50-FPN-RETINANET
resnet.StageSpec(index=1, block_count=3, return_features=True),
resnet.StageSpec(index=2, block_count=4, return_features=True),
resnet.StageSpec(index=3, block_count=6, return_features=True),
resnet.StageSpec(index=4, block_count=3, return_features=True)
]
transformation_module = resnet.BottleneckWithFixedBatchNorm
body = resnet.ResNetLight(stem_module,
stage_specs,
transformation_module,
freeze_conv_body_at=-1)
in_channels_p6p7 = in_channels_stage2 * 8 if use_c5 else out_channels
fpn = fpn_module.FPN(
in_channels_list=[
0,
in_channels_stage2 * 2,
in_channels_stage2 * 4,
in_channels_stage2 * 8,
],
out_channels=out_channels,
conv_block=conv_with_kaiming_uniform(use_gn, use_relu),
top_blocks=fpn_module.LastLevelP6P7(in_channels_p6p7, out_channels),
)
model = nn.Sequential(OrderedDict([("body", body), ("fpn", fpn)]))
model.out_channels = out_channels
return model
def build_dla_fpn_backbone(cfg):
body = dla(cfg)
in_channels_stage2 = cfg.MODEL.DLA.DLA_STAGE2_OUT_CHANNELS
in_channels_stage3 = cfg.MODEL.DLA.DLA_STAGE3_OUT_CHANNELS
in_channels_stage4 = cfg.MODEL.DLA.DLA_STAGE4_OUT_CHANNELS
in_channels_stage5 = cfg.MODEL.DLA.DLA_STAGE5_OUT_CHANNELS
out_channels = cfg.MODEL.DLA.BACKBONE_OUT_CHANNELS
fpn = fpn_module.FPN(
in_channels_list=[
in_channels_stage2,
in_channels_stage3,
in_channels_stage4,
in_channels_stage5
],
out_channels=out_channels,
conv_block=conv_with_kaiming_uniform(
cfg.MODEL.FPN.USE_GN, cfg.MODEL.FPN.USE_RELU
),
top_blocks=fpn_module.LastLevelMaxPool(),
)
model = nn.Sequential(OrderedDict([("body", body), ("fpn", fpn)]))
model.out_channels = out_channels
return model
def build_mnv2_fpn_backbone(cfg):
body = mobilenet.MobileNetV2(cfg)
in_channels_stage2 = cfg.MODEL.BACKBONE.ENCODER_OUT_CHANNELS
out_channels = cfg.MODEL.BACKBONE.OUT_CHANNELS
fpn = fpn_module.FPN(
in_channels_list=[
0,
in_channels_stage2[1],
in_channels_stage2[2],
in_channels_stage2[3],
],
out_channels=out_channels,
conv_block=conv_with_kaiming_uniform(cfg.MODEL.FPN.USE_GN,
cfg.MODEL.FPN.USE_RELU),
top_blocks=fpn_module.LastLevelP6P7(out_channels, out_channels),
)
if cfg.MODEL.BACKBONE.SPLIT:
# separate backbone and fpn output
return body, fpn
else:
model = nn.Sequential(OrderedDict([("body", body), ("fpn", fpn)]))
if cfg.MODEL.PANOPTIC.DECODER != "none":
return model, None
return model
def build_resnet_fpn_backbone(cfg):
in_channels_stage2 = cfg.MODEL.RESNETS.RES2_OUT_CHANNELS # 256
in_channels_list = [
in_channels_stage2, # 256
in_channels_stage2 * 2, # 512
in_channels_stage2 * 4, # 1024
in_channels_stage2 * 8, # 2048
]
body = resnet.ResNet(cfg)
out_channels = cfg.MODEL.RESNETS.BACKBONE_OUT_CHANNELS # 256
fpn = fpn_module.FPN(
in_channels_list=in_channels_list,
out_channels=out_channels,
conv_block=conv_with_kaiming_uniform(cfg.MODEL.FPN.USE_GN,
cfg.MODEL.FPN.USE_RELU,
cfg.MODEL.FPN.USE_DEFORMABLE),
top_blocks=fpn_module.LastLevelMaxPool(),
)
if cfg.MODEL.MSR_ON:
model = MSR(body, in_channels_list, fpn=fpn)
else:
model = nn.Sequential(OrderedDict([("body", body), ("fpn", fpn)]))
model.out_channels = out_channels
return model
def add_conv_body_fpn(cfg, dim_in=3):
builder, arch_def = create_builder(cfg)
body = FBNetTrunk(builder, arch_def, dim_in)
in_channels_stage2 = cfg.MODEL.BACKBONE.ENCODER_OUT_CHANNELS
out_channels = cfg.MODEL.BACKBONE.OUT_CHANNELS
fpn = fpn_module.FPN(
in_channels_list=[
0, in_channels_stage2[1], in_channels_stage2[2],
in_channels_stage2[3]
],
out_channels=out_channels,
conv_block=conv_with_kaiming_uniform(cfg.MODEL.FPN.USE_GN,
cfg.MODEL.FPN.USE_RELU),
top_blocks=fpn_module.LastLevelP6P7(out_channels, out_channels),
)
if cfg.MODEL.BACKBONE.SPLIT:
# separate backbone and fpn output
return body, fpn
else:
model = nn.Sequential(OrderedDict([("body", body), ("fpn", fpn)]))
if cfg.MODEL.PANOPTIC.DECODER != "none":
return model, None
return model
def __init__(self,
cfg,
in_channels_list,
in_channels_scale,
out_channels,
one_by_one_in_channels,
mode="bilinear"):
super(FPNBasedSemanticSegmentationHead, self).__init__()
self.mode = mode
self.upsampling_blocks = []
self.number_upsamples_per = []
priming = cfg.MODEL.RPN.USE_SEMANTIC_FEATURES or cfg.MODEL.ROI_HEADS.USE_SEMANTIC_FEATURES
# skip the possible "top" features?
target_scale = cfg.MODEL.SEMANTIC.COMBINE_AT_SCALE
for idx, in_channels in enumerate(in_channels_list):
upsampler_name = "upsample_scale{0}".format(idx)
in_channels = in_channels_list[idx]
scale = in_channels_scale[idx]
number_upsamples = int(np.log2(target_scale / scale))
self.number_upsamples_per.append(number_upsamples)
if number_upsamples == 0:
# paper is not quite clear what happens here. my guess is the usual but no upsampling.
upsampler = make_conv3x3(in_channels,
out_channels,
use_gn=cfg.MODEL.SEMANTIC.USE_GN,
use_relu=True)
# upsample1 = make_dfconv3x3(
# in_channels, out_channels, use_gn=False, use_relu=True)
# upsample2 = make_dfconv3x3(
# out_channels, out_channels, use_gn=False, use_relu=True)
# upsampler = nn.Sequential(*[upsample1, upsample2])
else:
upsampler = SetOfUpsamplingStages(cfg,
in_channels,
out_channels,
count=number_upsamples,
mode=self.mode)
#upsampler = StraightUpsamplingStages(cfg, in_channels, out_channels, count=number_upsamples, mode=self.mode)
self.add_module(upsampler_name, upsampler)
self.upsampling_blocks.append(upsampler)
if not cfg.MODEL.RPN.USE_SEMANTIC_FEATURES:
# unsure if there should be a ReLU here.
make_1x1_conv = conv_with_kaiming_uniform(use_gn=False,
use_relu=True)
self.conv = make_1x1_conv(one_by_one_in_channels,
out_channels,
kernel_size=1,
stride=1)
make_project = conv_with_kaiming_uniform(use_gn=False, use_relu=False)
# add VOID + THING vs VOID + THINGS + STUFF
number_classes = (1 + cfg.MODEL.SEMANTIC.NUM_CLASSES +
1 if cfg.MODEL.SEMANTIC.COLLAPSE_THING_ONTOLOGY else
cfg.MODEL.ROI_BOX_HEAD.NUM_CLASSES +
cfg.MODEL.SEMANTIC.NUM_CLASSES)
self.project = make_project(out_channels,
number_classes,
kernel_size=1,
stride=1)
