def build_fpn_backbone(cfg: CfgNode, input_shape: ShapeSpec) -> FPN:
"""
Build the Resnet 50 backbone and the FPN:
Label FPN:
* Input: C2, C3, C4
* Output: P2, P3, P4
Args:
cfg (CfgNode): A detectron2 CfgNode
input_shape (ShapeSpec): The input shape of the backbone.
Returns:
backbone (nn.Module): Backbone module, must be a subclass of :class:`Backbone`.
"""
# Build the feature extractor (Resnet 50)
bottom_up: ResNet = build_resnet_backbone(cfg, input_shape)
# Label FPN
label_in_features: List[str] = cfg.MODEL.FPN.IN_FEATURES
label_out_channels: List[str] = cfg.MODEL.FPN.OUT_CHANNELS
label_fpn: FPN = FPN(bottom_up=bottom_up,
in_features=label_in_features,
out_channels=label_out_channels,
norm=cfg.MODEL.FPN.NORM,
top_block=None,
fuse_type=cfg.MODEL.FPN.FUSE_TYPE)
return label_fpn
def build_resnet_fpn_wo_top_block_backbone(cfg, input_shape: ShapeSpec):
bottom_up = build_resnet_backbone(cfg, input_shape)
in_features = cfg.MODEL.FPN.IN_FEATURES
out_channels = cfg.MODEL.FPN.OUT_CHANNELS
backbone = FPN(bottom_up=bottom_up,
in_features=in_features,
out_channels=out_channels,
norm=cfg.MODEL.FPN.NORM,
fuse_type=cfg.MODEL.FPN.FUSE_TYPE)
return backbone
def __init__(self):
# Create model
shape = ShapeSpec(channels=3)
self.model = torch.nn.Sequential(
build_resnet_backbone(config.RESNET_CONFIG, shape))
# Load model weights
checkpointer = DetectionCheckpointer(self.model, save_to_disk=False)
checkpointer.load(load_remote_file(config.RESNET_MODEL_URL))
self.model.eval()
# Store relevant attributes of config
self.pixel_mean = torch.tensor(
config.RESNET_CONFIG.MODEL.PIXEL_MEAN).view(-1, 1, 1)
self.pixel_std = torch.tensor(
config.RESNET_CONFIG.MODEL.PIXEL_STD).view(-1, 1, 1)
self.input_format = config.RESNET_CONFIG.INPUT.FORMAT
def build_fpn_backbones(cfg: CfgNode,
input_shape: ShapeSpec) -> Tuple[FPN, FPN]:
"""
Build the Resnet 50 backbone and the two FPN networks:
* Panel FPN:
* Input: C3, C4, C5
* Output: P3, P4, P5, P6, P7
* Label FPN:
* Input: C2, C3, C4
* Output: P2, P3, P4
Args:
cfg (CfgNode): A detectron2 CfgNode.
input_shape (ShapeSpec): The input shape of the backbone.
Returns:
backbone (Backbone): Backbone module, must be a subclass of :class:`Backbone`.
"""
# Build the feature extractor (Resnet 50)
bottom_up: ResNet = build_resnet_backbone(cfg, input_shape)
# Panel FPN
panel_in_features: List[str] = cfg.MODEL.PANEL_FPN.IN_FEATURES
panel_out_channels: List[str] = cfg.MODEL.PANEL_FPN.OUT_CHANNELS
in_channels_p6p7: int = bottom_up.output_shape()['res5'].channels
panel_fpn: FPN = FPN(bottom_up=bottom_up,
in_features=panel_in_features,
out_channels=panel_out_channels,
norm=cfg.MODEL.FPN.NORM,
top_block=LastLevelP6P7(in_channels_p6p7,
panel_out_channels),
fuse_type=cfg.MODEL.FPN.FUSE_TYPE)
# Label FPN
label_in_features: List[str] = cfg.MODEL.LABEL_FPN.IN_FEATURES
label_out_channels: List[str] = cfg.MODEL.LABEL_FPN.OUT_CHANNELS
label_fpn: FPN = FPN(bottom_up=bottom_up,
in_features=label_in_features,
out_channels=label_out_channels,
norm=cfg.MODEL.FPN.NORM,
top_block=None,
fuse_type=cfg.MODEL.FPN.FUSE_TYPE)
return panel_fpn, label_fpn
def build_resnet_myfpn_backbone_v2(cfg, input_shape: ShapeSpec):
"""
Args:
cfg: a detectron2 CfgNode
Returns:
backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`.
"""
bottom_up = build_resnet_backbone(cfg, input_shape)
in_features = cfg.MODEL.FPN.IN_FEATURES
out_channels = cfg.MODEL.FPN.OUT_CHANNELS
backbone = MYFPN(
bottom_up=bottom_up,
in_features=in_features,
out_channels=out_channels,
norm=cfg.MODEL.FPN.NORM,
top_block=LastLevelMaxPoolV2(out_channels, sync_bn=cfg.MODEL.SYNC_BN),
fuse_type=cfg.MODEL.FPN.FUSE_TYPE,
)
return backbone
def build_p35_resnet_fpn_backbone(cfg, input_shape: ShapeSpec):
"""
Args:
cfg: a detectron2 CfgNode
Returns:
backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`.
"""
bottom_up = build_resnet_backbone(cfg, input_shape)
in_features = cfg.MODEL.FPN.IN_FEATURES
out_channels = cfg.MODEL.FPN.OUT_CHANNELS
backbone = FPN(
bottom_up=bottom_up,
in_features=in_features,
out_channels=out_channels,
norm=cfg.MODEL.FPN.NORM,
top_block=None,
fuse_type=cfg.MODEL.FPN.FUSE_TYPE,
)
return backbone
def initialize_container(self):
# Create model
shape = ShapeSpec(channels=3)
self.model = torch.nn.Sequential(
build_resnet_backbone(config.RESNET_CONFIG, shape))
# Load model weights
checkpointer = DetectionCheckpointer(self.model, save_to_disk=False)
checkpointer.load(config.WEIGHTS_PATH)
self.model.eval()
# Store relevant attributes of config
self.pixel_mean = torch.tensor(
config.RESNET_CONFIG.MODEL.PIXEL_MEAN).view(-1, 1, 1)
self.pixel_std = torch.tensor(
config.RESNET_CONFIG.MODEL.PIXEL_STD).view(-1, 1, 1)
self.input_format = config.RESNET_CONFIG.INPUT.FORMAT
# Create connection pool
self.session = aiohttp.ClientSession()
def build_resnet_bifpn_backbone(cfg, input_shape: ShapeSpec):
"""
Args:
cfg: a detectron2 CfgNode
Returns:
backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`.
"""
bottom_up = build_resnet_backbone(cfg, input_shape)
in_features = cfg.MODEL.FPN.IN_FEATURES
backbone = BiFPN(
cfg=cfg,
bottom_up=bottom_up,
in_features=in_features,
out_channels=cfg.MODEL.BIFPN.OUT_CHANNELS,
norm=cfg.MODEL.BIFPN.NORM,
num_levels=cfg.MODEL.BIFPN.NUM_LEVELS,
num_bifpn=cfg.MODEL.BIFPN.NUM_BIFPN,
separable_conv=cfg.MODEL.BIFPN.SEPARABLE_CONV,
)
return backbone
def build_retinanet_resnet_fpn_backbone(cfg, input_shape: ShapeSpec):
"""
Args:
cfg: a detectron2 CfgNode
Returns:
backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`.
"""
bottom_up = build_resnet_backbone(cfg, input_shape)
in_features = cfg.MODEL.FPN.IN_FEATURES
out_channels = cfg.MODEL.FPN.OUT_CHANNELS
in_channels_p6p7 = bottom_up.output_shape()["res5"].channels
backbone = FPN(
bottom_up=bottom_up,
in_features=in_features,
out_channels=out_channels,
norm=cfg.MODEL.FPN.NORM,
top_block=LastLevelP6P7(in_channels_p6p7, out_channels),
fuse_type=cfg.MODEL.FPN.FUSE_TYPE,
)
return backbone
def init_model(self):
assert self.backbone in [
"resnet18", "resnet50", "shufflenet_v2_x1_0", "resnet50_detectron"
]
detectron_resnet_layer4 = None
if self.backbone == "resnet18":
backbone = resnet18
backbone_network = backbone(first_conv=self.first_conv,
maxpool1=self.maxpool1,
return_all_feature_maps=False)
self.feature_dim = backbone_network.fc.in_features
elif self.backbone == "resnet50":
backbone = resnet50
backbone_network = backbone(first_conv=self.first_conv,
maxpool1=self.maxpool1,
return_all_feature_maps=False)
self.feature_dim = backbone_network.fc.in_features
elif self.backbone == "shufflenet_v2_x1_0":
backbone = shufflenet_v2_x1_0
backbone_network = backbone()
self.feature_dim = backbone_network.fc.in_features
backbone_network.fc = Identity()
elif self.backbone == "resnet50_detectron":
with open("examples/local/detectron_resnet50_c4_config.yaml",
"r") as f:
import yaml
cfg = yaml.load(f, Loader=yaml.Loader)
from detectron2.modeling.backbone.resnet import build_resnet_backbone
from detectron2.layers import ShapeSpec
input_shape = ShapeSpec(3) #3 channels RGB
backbone_network = build_resnet_backbone(cfg, input_shape)
backbone_network = unfreeze_batchnorm_layers(backbone_network)
detectron_resnet_layer4 = Resnet50Layer4()
self.feature_dim = 2048
else:
raise ValueError(f"Unsupported backbone: {self.backbone}")
if self.coordconv is not None:
from thelper.nn.coordconv import swap_coordconv_layers #Lazy loading.
if self.coordconv == "all":
backbone_network = swap_coordconv_layers(backbone_network)
if self.coordconv == "first":
backbone_network.conv1 = swap_coordconv_layers(
backbone_network.conv1)
#backbone_network =
self.cyclic_predictor = None
if self.loss_function == "cyclic":
#Use 2 stacked inputs for the predictor
self.cyclic_predictor = PredictionMLP(self.feature_dim * 2,
self.hidden_mlp,
self.feature_dim)
#else:
#All other methods work on pairs!
self.online_network = SiameseArm(
backbone_network,
input_dim=self.feature_dim,
hidden_size=self.hidden_mlp,
output_dim=self.feat_dim,
detectron_resnet_layer4=detectron_resnet_layer4)
#max_batch = math.ceil(self.num_samples/self.batch_size)
encoder, projector = self.online_network.encoder, self.online_network.projector
self.train_features = torch.zeros((self.num_samples, self.feature_dim))
self.train_meta = []
self.train_targets = -torch.ones((self.num_samples))
self.valid_features = torch.zeros(
(self.num_samples_valid, self.feature_dim))
self.valid_meta = []
self.cuda_train_features = None
