def __init__(self):
super().__init__()
# For cache
self.mapping_to_detectron = None
self.orphans_in_detectron = None
# Backbone for feature extraction
self.Conv_Body = get_func(cfg.MODEL.CONV_BODY)()
self.Box_Head = get_func(cfg.FAST_RCNN.ROI_BOX_HEAD)(
self.Conv_Body.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
self.Box_MIL_Outs = pcl_heads.mil_outputs(self.Box_Head.dim_out,
cfg.MODEL.NUM_CLASSES)
self.Box_Refine_Outs = pcl_heads.refine_outputs(
self.Box_Head.dim_out, cfg.MODEL.NUM_CLASSES + 1)
self.Refine_Losses = [OICRLosses() for i in range(cfg.REFINE_TIMES)]
if cfg.MODEL.WITH_FRCNN:
self.FRCNN_Outs = fast_rcnn_heads.fast_rcnn_outputs(
self.Box_Head.dim_out, cfg.MODEL.NUM_CLASSES + 1)
self.Cls_Loss = OICRLosses()
self._init_modules()
def __init__(self):
super().__init__()
# For cache
self.mapping_to_detectron = None
self.orphans_in_detectron = None
# Backbone for feature extraction
self.Conv_Body = get_func(cfg.MODEL.CONV_BODY)()
# Region Proposal Network
if cfg.RPN.RPN_ON:
self.RPN = rpn_heads.generic_rpn_outputs(
self.Conv_Body.dim_out, self.Conv_Body.spatial_scale)
if cfg.FPN.FPN_ON:
# Only supports case when RPN and ROI min levels are the same
assert cfg.FPN.RPN_MIN_LEVEL == cfg.FPN.ROI_MIN_LEVEL
# RPN max level can be >= to ROI max level
assert cfg.FPN.RPN_MAX_LEVEL >= cfg.FPN.ROI_MAX_LEVEL
# FPN RPN max level might be > FPN ROI max level in which case we
# need to discard some leading conv blobs (blobs are ordered from
# max/coarsest level to min/finest level)
self.num_roi_levels = cfg.FPN.ROI_MAX_LEVEL - cfg.FPN.ROI_MIN_LEVEL + 1
# Retain only the spatial scales that will be used for RoI heads. `Conv_Body.spatial_scale`
# may include extra scales that are used for RPN proposals, but not for RoI heads.
self.Conv_Body.spatial_scale = self.Conv_Body.spatial_scale[-self.num_roi_levels:]
# BBOX Branch #hw RPN output to ROIPool layer
if not cfg.MODEL.RPN_ONLY:
if cfg.MODEL.FASTER_RCNN:
self.Box_Head = get_func(cfg.FAST_RCNN.ROI_BOX_HEAD)(
# hw ResNet_roi_conv5_head RPN网络输出roi和prob到ROIAlgin层
self.RPN.dim_out, self.roi_feature_transform, self.Conv_Body.spatial_scale)
self.Box_Outs = fast_rcnn_heads.fast_rcnn_outputs(#hw 最后输出avgpool并输入到进入全连接层输出box和prob
self.Box_Head.dim_out)
elif cfg.MODEL.LIGHT_HEAD_RCNN:#hw
from modeling.light_head_rcnn_heads import ResNet_Conv5_light_head
self.LightHead = ResNet_Conv5_light_head(self.Conv_Body.dim_out)
self.Box_Outs = light_head_rcnn_heads.light_head_rcnn_outputs(490,self.roi_feature_transform,self.Conv_Body.spatial_scale)
# Mask Branch
if cfg.MODEL.MASK_ON:
self.Mask_Head = get_func(cfg.MRCNN.ROI_MASK_HEAD)(
self.RPN.dim_out, self.roi_feature_transform, self.Conv_Body.spatial_scale)
if getattr(self.Mask_Head, 'SHARE_RES5', False):
self.Mask_Head.share_res5_module(self.Box_Head.res5)
self.Mask_Outs = mask_rcnn_heads.mask_rcnn_outputs(self.Mask_Head.dim_out)
# Keypoints Branch
if cfg.MODEL.KEYPOINTS_ON:
self.Keypoint_Head = get_func(cfg.KRCNN.ROI_KEYPOINTS_HEAD)(
self.RPN.dim_out, self.roi_feature_transform, self.Conv_Body.spatial_scale)
if getattr(self.Keypoint_Head, 'SHARE_RES5', False):
self.Keypoint_Head.share_res5_module(self.Box_Head.res5)
self.Keypoint_Outs = keypoint_rcnn_heads.keypoint_outputs(self.Keypoint_Head.dim_out)
self._init_modules()
def __init__(self):
super().__init__()
# For cache
self.mapping_to_detectron = None
self.orphans_in_detectron = None
# Backbone for feature extraction
self.Conv_Body = get_func(cfg.MODEL.CONV_BODY)()
# Region Proposal Network
if cfg.RPN.RPN_ON:
self.RPN = rpn_heads.generic_rpn_outputs(
self.Conv_Body.dim_out, self.Conv_Body.spatial_scale)
if cfg.FPN.FPN_ON:
# Only supports case when RPN and ROI min levels are the same
assert cfg.FPN.RPN_MIN_LEVEL == cfg.FPN.ROI_MIN_LEVEL
# RPN max level can be >= to ROI max level
assert cfg.FPN.RPN_MAX_LEVEL >= cfg.FPN.ROI_MAX_LEVEL
# FPN RPN max level might be > FPN ROI max level in which case we
# need to discard some leading conv blobs (blobs are ordered from
# max/coarsest level to min/finest level)
self.num_roi_levels = cfg.FPN.ROI_MAX_LEVEL - cfg.FPN.ROI_MIN_LEVEL + 1
# Retain only the spatial scales that will be used for RoI heads. `Conv_Body.spatial_scale`
# may include extra scales that are used for RPN proposals, but not for RoI heads.
self.Conv_Body.spatial_scale = self.Conv_Body.spatial_scale[
-self.num_roi_levels:]
# BBOX Branch
if not cfg.MODEL.RPN_ONLY:
self.Box_Head = get_func(cfg.FAST_RCNN.ROI_BOX_HEAD)(
self.RPN.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
self.Box_Outs = fast_rcnn_heads.fast_rcnn_outputs(
self.Box_Head.dim_out)
# Mask Branch
if cfg.MODEL.MASK_ON:
self.Amodal_Mask_Head = get_func(cfg.MRCNN.ROI_MASK_HEAD)(
self.RPN.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
if getattr(self.Amodal_Mask_Head, 'SHARE_RES5', False):
self.Amodal_Mask_Head.share_res5_module(self.Box_Head.res5)
self.Amodal_Mask_Outs = mask_rcnn_heads.mask_rcnn_outputs(
self.Amodal_Mask_Head.dim_out)
# Inmodal Branch
if cfg.MODEL.INMODAL_ON:
self.Inmodal_Mask_Head = get_func(cfg.MRCNN.ROI_MASK_HEAD)(
self.RPN.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
if getattr(self.Inmodal_Mask_Head, 'SHARE_RES5', False):
self.Inmodal_Mask_Head.share_res5_module(self.Box_Head.res5)
self.Inmodal_Mask_Outs = mask_rcnn_heads.mask_rcnn_outputs(
self.Inmodal_Mask_Head.dim_out)
self._init_modules()
def __init__(self):
super().__init__()
# For cache
self.mapping_to_detectron = None
self.orphans_in_detectron = None
self.use_gt_boxes = True # a temporary setting
# Backbone for feature extraction
self.Conv_Body = get_func(cfg.MODEL.CONV_BODY)()
# Region Proposal Network
if not self.use_gt_boxes and cfg.RPN.RPN_ON:
self.RPN = rpn_heads.generic_rpn_outputs(
self.Conv_Body.dim_out, self.Conv_Body.spatial_scale)
if cfg.FPN.FPN_ON:
# Only supports case when RPN and ROI min levels are the same
assert cfg.FPN.RPN_MIN_LEVEL == cfg.FPN.ROI_MIN_LEVEL
# RPN max level can be >= to ROI max level
assert cfg.FPN.RPN_MAX_LEVEL >= cfg.FPN.ROI_MAX_LEVEL
# FPN RPN max level might be > FPN ROI max level in which case we
# need to discard some leading conv blobs (blobs are ordered from
# max/coarsest level to min/finest level)
self.num_roi_levels = cfg.FPN.ROI_MAX_LEVEL - cfg.FPN.ROI_MIN_LEVEL + 1
# Retain only the spatial scales that will be used for RoI heads. `Conv_Body.spatial_scale`
# may include extra scales that are used for RPN proposals, but not for RoI heads.
self.Conv_Body.spatial_scale = self.Conv_Body.spatial_scale[
-self.num_roi_levels:]
# BBOX Branch
self.Box_Head = get_func(cfg.FAST_RCNN.ROI_BOX_HEAD)(
self.Conv_Body.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
# self.RPN.dim_out, self.roi_feature_transform, self.Conv_Body.spatial_scale)
if not self.use_gt_boxes:
self.Box_Outs = fast_rcnn_heads.fast_rcnn_outputs(
self.Box_Head.dim_out)
self.Prd_RCNN = copy.deepcopy(self)
if not self.use_gt_boxes:
del self.Prd_RCNN.RPN
del self.Prd_RCNN.Box_Outs
# initialize word vectors
ds_name = cfg.TRAIN.DATASETS[0] if len(
cfg.TRAIN.DATASETS) else cfg.TEST.DATASETS[0]
self.obj_vecs, self.prd_vecs = get_obj_prd_vecs(ds_name)
# RelPN
self.RelPN = relpn_heads.generic_relpn_outputs()
# RelDN
self.RelDN = reldn_heads.reldn_head(self.Box_Head.dim_out * 3,
self.obj_vecs,
self.prd_vecs) # concat of SPO
self._init_modules()
def __init__(self):
super().__init__()
# For cache
self.mapping_to_detectron = None
self.orphans_in_detectron = None
# Backbone for feature extraction
self.Conv_Body = get_func(cfg.MODEL.CONV_BODY)()
# Region Proposal Network
if cfg.RPN.RPN_ON:
self.RPN = rpn_heads.generic_rpn_outputs(
self.Conv_Body.dim_out, self.Conv_Body.spatial_scale)
if cfg.FPN.FPN_ON:
# Only supports case when RPN and ROI min levels are the same
assert cfg.FPN.RPN_MIN_LEVEL == cfg.FPN.ROI_MIN_LEVEL
# RPN max level can be >= to ROI max level
assert cfg.FPN.RPN_MAX_LEVEL >= cfg.FPN.ROI_MAX_LEVEL
# FPN RPN max level might be > FPN ROI max level in which case we
# need to discard some leading conv blobs (blobs are ordered from
# max/coarsest level to min/finest level)
self.num_roi_levels = cfg.FPN.ROI_MAX_LEVEL - cfg.FPN.ROI_MIN_LEVEL + 1
# Retain only the spatial scales that will be used for RoI heads. `Conv_Body.spatial_scale`
# may include extra scales that are used for RPN proposals, but not for RoI heads.
self.Conv_Body.spatial_scale = self.Conv_Body.spatial_scale[-self.num_roi_levels:]
# BBOX Branch
if not cfg.MODEL.RPN_ONLY:
self.Box_Head = get_func(cfg.FAST_RCNN.ROI_BOX_HEAD)(
self.RPN.dim_out, self.roi_feature_transform, self.Conv_Body.spatial_scale)
self.Box_Outs = fast_rcnn_heads.fast_rcnn_outputs(
self.Box_Head.dim_out)
# Mask Branch
if cfg.MODEL.MASK_ON:
self.Mask_Head = get_func(cfg.MRCNN.ROI_MASK_HEAD)(
self.RPN.dim_out, self.roi_feature_transform, self.Conv_Body.spatial_scale)
if getattr(self.Mask_Head, 'SHARE_RES5', False):
self.Mask_Head.share_res5_module(self.Box_Head.res5)
self.Mask_Outs = mask_rcnn_heads.mask_rcnn_outputs(self.Mask_Head.dim_out)
# Keypoints Branch
if cfg.MODEL.KEYPOINTS_ON:
self.Keypoint_Head = get_func(cfg.KRCNN.ROI_KEYPOINTS_HEAD)(
self.RPN.dim_out, self.roi_feature_transform, self.Conv_Body.spatial_scale)
if getattr(self.Keypoint_Head, 'SHARE_RES5', False):
self.Keypoint_Head.share_res5_module(self.Box_Head.res5)
self.Keypoint_Outs = keypoint_rcnn_heads.keypoint_outputs(self.Keypoint_Head.dim_out)
self._init_modules()
def __init__(self):
super().__init__()
if cfg.LESION.USE_POSITION:
#self.Position_Head = position_Xconv1fc_gn_head(2048,1024,3)
self.Position_Head = position_Xconv1fc_gn_head(256, 1024, 3)
self.Position_Cls_Outs = position_cls_outputs(
self.Position_Head.dim_out)
self.Position_Reg_Outs = position_reg_outputs(
self.Position_Head.dim_out)
if cfg.LESION.POS_CONCAT_RCNN:
self.Box_Outs = fast_rcnn_heads.fast_rcnn_outputs(
self.Box_Head.dim_out + self.Position_Head.dim_out)
self.cbam = CBAM(self.Conv_Body.dim_out * cfg.LESION.NUM_IMAGES_3DCE,
16,
no_spatial=True)
def __init__(self, dim_in, resolution, pretrained_weights=None):
super().__init__()
self.fc_dim = dim_in * resolution * resolution
self.mapping_to_detectron = None
self.orphans_in_detectron = None
self.adversarial = nn.Sequential(nn.Linear(self.fc_dim, 4096),
nn.LeakyReLU(negative_slope=0.2),
nn.Linear(4096, 1024),
nn.LeakyReLU(negative_slope=0.2),
nn.Linear(1024, 1), nn.Sigmoid())
self.adversarial_criterion = nn.BCELoss()
self._init_weights()
self.Box_Head = get_func(cfg.GAN.MODEL.CONV_BODY_FC_HEAD)(dim_in,
resolution)
self.Box_Outs = fast_rcnn_heads.fast_rcnn_outputs(
self.Box_Head.dim_out)
self._init_modules(pretrained_weights)
def __init__(self):
super().__init__()
# For cache
self.mapping_to_detectron = None
self.orphans_in_detectron = None
# Backbone for feature extraction
self.Conv_Body = get_func(cfg.MODEL.CONV_BODY)()
# Region Proposal Network
if not cfg.TRAIN.USE_GT_BOXES and cfg.RPN.RPN_ON:
self.RPN = rpn_heads.generic_rpn_outputs(
self.Conv_Body.dim_out, self.Conv_Body.spatial_scale)
if cfg.FPN.FPN_ON:
# Only supports case when RPN and ROI min levels are the same
assert cfg.FPN.RPN_MIN_LEVEL == cfg.FPN.ROI_MIN_LEVEL
# RPN max level can be >= to ROI max level
assert cfg.FPN.RPN_MAX_LEVEL >= cfg.FPN.ROI_MAX_LEVEL
# FPN RPN max level might be > FPN ROI max level in which case we
# need to discard some leading conv blobs (blobs are ordered from
# max/coarsest level to min/finest level)
self.num_roi_levels = cfg.FPN.ROI_MAX_LEVEL - cfg.FPN.ROI_MIN_LEVEL + 1
# Retain only the spatial scales that will be used for RoI heads. `Conv_Body.spatial_scale`
# may include extra scales that are used for RPN proposals, but not for RoI heads.
self.Conv_Body.spatial_scale = self.Conv_Body.spatial_scale[
-self.num_roi_levels:]
# BBOX Branch
self.Box_Head = get_func(cfg.FAST_RCNN.ROI_BOX_HEAD)(
self.Conv_Body.dim_out, 4096, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
# self.RPN.dim_out, self.roi_feature_transform, self.Conv_Body.spatial_scale)
if not cfg.TRAIN.USE_GT_BOXES:
self.Box_Outs = fast_rcnn_heads.fast_rcnn_outputs(
self.Box_Head.dim_out)
self.Prd_RCNN = copy.deepcopy(self)
if not cfg.TRAIN.USE_GT_BOXES:
del self.Prd_RCNN.RPN
del self.Prd_RCNN.Box_Outs
# initialize word vectors
ds_name = cfg.TRAIN.DATASETS[0] if len(
cfg.TRAIN.DATASETS) else cfg.TEST.DATASETS[0]
obj_categories, prd_categories = get_obj_prd_vecs(ds_name)
# RelPN
self.RelPN = relpn_heads.generic_relpn_outputs()
# RelDN
self.RelDN = reldn_heads.reldn_head(self.Box_Head.dim_out *
3) # concat of SPO
self.prd_weights = None
self.obj_weights = None
if cfg.DATASET == 'gvqa10k':
freq_prd_path = cfg.DATA_DIR + '/gvqa/reduced_data/10k/seed{}/predicates_freqs.json'.format(
cfg.RNG_SEED)
freq_obj_path = cfg.DATA_DIR + '/gvqa/reduced_data/10k/seed{}/objects_freqs.json'.format(
cfg.RNG_SEED)
elif cfg.DATASET == 'gvqa20k':
freq_prd_path = cfg.DATA_DIR + '/gvqa/reduced_data/20k/seed{}/predicates_freqs.json'.format(
cfg.RNG_SEED)
freq_obj_path = cfg.DATA_DIR + '/gvqa/reduced_data/20k/seed{}/objects_freqs.json'.format(
cfg.RNG_SEED)
elif cfg.DATASET == 'gvqa':
freq_prd_path = cfg.DATA_DIR + '/gvqa/seed{}/predicates_freqs.json'.format(
cfg.RNG_SEED)
freq_obj_path = cfg.DATA_DIR + '/gvqa/seed{}/objects_freqs.json'.format(
cfg.RNG_SEED)
elif cfg.DATASET == 'vg80k':
freq_prd_path = cfg.DATA_DIR + '/vg/predicates_freqs.json'
freq_obj_path = cfg.DATA_DIR + '/vg/objects_freqs.json'
elif cfg.DATASET == 'vg8k':
freq_prd_path = cfg.DATA_DIR + '/vg8k/seed{}/train_predicates_freqs.json'.format(
cfg.RNG_SEED)
freq_obj_path = cfg.DATA_DIR + '/vg8k/seed{}/train_objects_freqs.json'.format(
cfg.RNG_SEED)
else:
raise NotImplementedError
self.prd_freq_dict = json.load(open(freq_prd_path))
self.obj_freq_dict = json.load(open(freq_obj_path))
no_bg_prd_categories = prd_categories[1:]
assert len(no_bg_prd_categories) == cfg.MODEL.NUM_PRD_CLASSES
self.prd_categories = no_bg_prd_categories
self.obj_categories = obj_categories
self.freq_prd = get_freq_from_dict(self.prd_freq_dict,
self.prd_categories)
self.freq_obj = get_freq_from_dict(self.obj_freq_dict,
self.obj_categories)
if cfg.MODEL.LOSS == 'weighted_cross_entropy':
logger.info('loading frequencies')
freq_prd = self.freq_prd + 1
freq_obj = self.freq_obj + 1
prd_weights = np.sum(freq_prd) / freq_prd
obj_weights = np.sum(freq_obj) / freq_obj
self.prd_weights = (prd_weights / np.mean(prd_weights)).astype(
np.float32)
self.obj_weights = (obj_weights / np.mean(obj_weights)).astype(
np.float32)
temp = np.zeros(shape=self.prd_weights.shape[0] + 1,
dtype=np.float32)
temp[1:] = self.prd_weights
temp[0] = min(self.prd_weights)
self.prd_weights = temp
self._init_modules()
def __init__(self):
super().__init__()
# For cache
self.mapping_to_detectron = None
self.orphans_in_detectron = None
# Backbone for feature extraction
'''
FPN.fpn_ResNet50_conv5_body for example
get_func will get the function by name
here is where FPN_ResNet lay functions are called
It returns a ResNet that has been associated with FPN
'''
self.Conv_Body = get_func(cfg.MODEL.CONV_BODY)()
# Region Proposal Network
'''
Fast RCNN will have RPN on according to config.py
HERE IS WHERE THE RPN CREATED
'''
if cfg.RPN.RPN_ON:
'''
generic_rpn_outputs calls rpn_heads.FPN.fpn_rpn_outputs,
which creates the head subnets, builds anchor of different levels
'''
self.RPN = rpn_heads.generic_rpn_outputs(
self.Conv_Body.dim_out, self.Conv_Body.spatial_scale)
if cfg.FPN.FPN_ON:
# Only supports case when RPN and ROI min levels are the same
assert cfg.FPN.RPN_MIN_LEVEL == cfg.FPN.ROI_MIN_LEVEL
# RPN max level can be >= to ROI max level
assert cfg.FPN.RPN_MAX_LEVEL >= cfg.FPN.ROI_MAX_LEVEL
# FPN RPN max level might be > FPN ROI max level in which case we
# need to discard some leading conv blobs (blobs are ordered from
# max/coarsest level to min/finest level)
self.num_roi_levels = cfg.FPN.ROI_MAX_LEVEL - cfg.FPN.ROI_MIN_LEVEL + 1
# Retain only the spatial scales that will be used for RoI heads. `Conv_Body.spatial_scale`
# may include extra scales that are used for RPN proposals, but not for RoI heads.
self.Conv_Body.spatial_scale = self.Conv_Body.spatial_scale[
-self.num_roi_levels:]
# BBOX Branch
'''when created, RPN_ONLY is set to false by default'''
if not cfg.MODEL.RPN_ONLY:
'''in faster rcnn fpn r-50 case it is fast_rcnn_heads.roi_2mlp_head'''
self.Box_Head = get_func(cfg.FAST_RCNN.ROI_BOX_HEAD)(
self.RPN.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
self.Box_Outs = fast_rcnn_heads.fast_rcnn_outputs(
self.Box_Head.dim_out)
# Mask Branch
if cfg.MODEL.MASK_ON:
self.Mask_Head = get_func(cfg.MRCNN.ROI_MASK_HEAD)(
self.RPN.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
if getattr(self.Mask_Head, 'SHARE_RES5', False):
self.Mask_Head.share_res5_module(self.Box_Head.res5)
self.Mask_Outs = mask_rcnn_heads.mask_rcnn_outputs(
self.Mask_Head.dim_out)
# Keypoints Branch
if cfg.MODEL.KEYPOINTS_ON:
self.Keypoint_Head = get_func(cfg.KRCNN.ROI_KEYPOINTS_HEAD)(
self.RPN.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
if getattr(self.Keypoint_Head, 'SHARE_RES5', False):
self.Keypoint_Head.share_res5_module(self.Box_Head.res5)
self.Keypoint_Outs = keypoint_rcnn_heads.keypoint_outputs(
self.Keypoint_Head.dim_out)
self._init_modules()
def __init__(self, ds=None):
super().__init__()
# For cache
self.mapping_to_detectron = None
self.orphans_in_detectron = None
# Backbone for feature extraction
self.Conv_Body = get_func(cfg.MODEL.CONV_BODY)()
# Region Proposal Network
if cfg.RPN.RPN_ON:
self.RPN = rpn_heads.generic_rpn_outputs(
self.Conv_Body.dim_out, self.Conv_Body.spatial_scale)
if cfg.FPN.FPN_ON:
# Only supports case when RPN and ROI min levels are the same
assert cfg.FPN.RPN_MIN_LEVEL == cfg.FPN.ROI_MIN_LEVEL
# RPN max level can be >= to ROI max level
assert cfg.FPN.RPN_MAX_LEVEL >= cfg.FPN.ROI_MAX_LEVEL
# FPN RPN max level might be > FPN ROI max level in which case we
# need to discard some leading conv blobs (blobs are ordered from
# max/coarsest level to min/finest level)
self.num_roi_levels = cfg.FPN.ROI_MAX_LEVEL - cfg.FPN.ROI_MIN_LEVEL + 1
# Retain only the spatial scales that will be used for RoI heads. `Conv_Body.spatial_scale`
# may include extra scales that are used for RPN proposals, but not for RoI heads.
self.Conv_Body.spatial_scale = self.Conv_Body.spatial_scale[
-self.num_roi_levels:]
# BBOX Branch
if not cfg.MODEL.RPN_ONLY:
self.Box_Head = get_func(cfg.FAST_RCNN.ROI_BOX_HEAD)(
self.RPN.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
self.Box_Outs = fast_rcnn_heads.fast_rcnn_outputs(
self.Box_Head.dim_out)
# BBOX Branch for finer car model classification
if cfg.MODEL.CAR_CLS_HEAD_ON:
self.car_cls_Head = get_func(cfg.CAR_CLS.ROI_BOX_HEAD)(
self.RPN.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
self.car_cls_Outs = car_3d_pose_heads.fast_rcnn_outputs_car_cls_rot(
self.car_cls_Head.dim_out)
self.shape_sim_mat = np.loadtxt('./utilities/sim_mat.txt')
# TRANS Branch for car translation regression
if cfg.MODEL.TRANS_HEAD_ON:
if cfg.TRANS_HEAD.INPUT_CONV_BODY:
self.car_trans_Head = get_func(cfg.TRANS_HEAD.TRANS_HEAD)(
self.RPN.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale, cfg.TRANS_HEAD.INPUT_DIM)
self.car_trans_Outs = car_3d_pose_heads.car_trans_outputs(
self.car_trans_Head.dim_out)
elif cfg.TRANS_HEAD.INPUT_TRIPLE_HEAD:
# We use the 1024 dim from car_cls+rot head
self.car_trans_Head = get_func(cfg.TRANS_HEAD.TRANS_HEAD)(
cfg.TRANS_HEAD.INPUT_DIM)
self.car_trans_Outs = car_3d_pose_heads.car_trans_triple_outputs(
self.car_trans_Head.dim_out, self.car_cls_Head.dim_out)
else:
self.car_trans_Head = get_func(cfg.TRANS_HEAD.TRANS_HEAD)(
cfg.TRANS_HEAD.INPUT_DIM)
self.car_trans_Outs = car_3d_pose_heads.car_trans_outputs(
self.car_trans_Head.dim_out)
# 3D to 2D projection error for multi-loss
if cfg.MODEL.LOSS_3D_2D_ON:
self.car_models = ds.load_car_models()
self.car_names = ds.unique_car_names
self.intrinsic_mat = ds.get_intrinsic_mat()
# Mask Branch
if cfg.MODEL.MASK_ON:
self.Mask_Head = get_func(cfg.MRCNN.ROI_MASK_HEAD)(
self.RPN.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
if getattr(self.Mask_Head, 'SHARE_RES5', False):
self.Mask_Head.share_res5_module(self.Box_Head.res5)
self.Mask_Outs = mask_rcnn_heads.mask_rcnn_outputs(
self.Mask_Head.dim_out)
# Keypoints Branch
if cfg.MODEL.KEYPOINTS_ON:
self.Keypoint_Head = get_func(cfg.KRCNN.ROI_KEYPOINTS_HEAD)(
self.RPN.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
if getattr(self.Keypoint_Head, 'SHARE_RES5', False):
self.Keypoint_Head.share_res5_module(self.Box_Head.res5)
self.Keypoint_Outs = keypoint_rcnn_heads.keypoint_outputs(
self.Keypoint_Head.dim_out)
self._init_modules()
def __init__(self):
super().__init__()
print("import {}".format(cfg.NETWORK_NAME))
if cfg.NETWORK_NAME == 'PMFNet_Baseline':
from modeling.hoi import PMFNet_Baseline as HOI
elif cfg.NETWORK_NAME == 'PMFNet_Final':
from modeling.hoi import PMFNet_Final as HOI
# For cache
self.mapping_to_detectron = None
self.orphans_in_detectron = None
# Backbone for feature extraction
self.Conv_Body = get_func(cfg.MODEL.CONV_BODY)()
# Region Proposal Network
if cfg.RPN.RPN_ON:
self.RPN = rpn_heads.generic_rpn_outputs(
self.Conv_Body.dim_out, self.Conv_Body.spatial_scale)
if cfg.FPN.FPN_ON:
# Only supports case when RPN and ROI min levels are the same
assert cfg.FPN.RPN_MIN_LEVEL == cfg.FPN.ROI_MIN_LEVEL
# RPN max level can be >= to ROI max level
assert cfg.FPN.RPN_MAX_LEVEL >= cfg.FPN.ROI_MAX_LEVEL
# FPN RPN max level might be > FPN ROI max level in which case we
# need to discard some leading conv blobs (blobs are ordered from
# max/coarsest level to min/finest level)
self.num_roi_levels = cfg.FPN.ROI_MAX_LEVEL - cfg.FPN.ROI_MIN_LEVEL + 1
# Retain only the spatial scales that will be used for RoI heads. `Conv_Body.spatial_scale`
# may include extra scales that are used for RPN proposals, but not for RoI heads.
self.Conv_Body.spatial_scale = self.Conv_Body.spatial_scale[
-self.num_roi_levels:]
# BBOX Branch
if not cfg.MODEL.RPN_ONLY:
self.Box_Head = get_func(cfg.FAST_RCNN.ROI_BOX_HEAD)(
self.RPN.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
self.Box_Outs = fast_rcnn_heads.fast_rcnn_outputs(
self.Box_Head.dim_out)
if cfg.MODEL.VCOCO_ON:
if cfg.FPN.MULTILEVEL_ROIS:
self.hoi_spatial_scale = self.Conv_Body.spatial_scale
else:
self.hoi_spatial_scale = self.Conv_Body.spatial_scale[-1]
self.HOI_Head = HOI(self.Conv_Body.dim_out,
self.roi_feature_transform,
self.hoi_spatial_scale)
# Mask Branch
if cfg.MODEL.MASK_ON:
self.Mask_Head = get_func(cfg.MRCNN.ROI_MASK_HEAD)(
self.RPN.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
if getattr(self.Mask_Head, 'SHARE_RES5', False):
self.Mask_Head.share_res5_module(self.Box_Head.res5)
self.Mask_Outs = mask_rcnn_heads.mask_rcnn_outputs(
self.Mask_Head.dim_out)
# Keypoints Branch
if cfg.MODEL.KEYPOINTS_ON:
self.Keypoint_Head = get_func(cfg.KRCNN.ROI_KEYPOINTS_HEAD)(
self.RPN.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
if getattr(self.Keypoint_Head, 'SHARE_RES5', False):
self.Keypoint_Head.share_res5_module(self.Box_Head.res5)
self.Keypoint_Outs = keypoint_rcnn_heads.keypoint_outputs(
self.Keypoint_Head.dim_out)
self._init_modules()
def __init__(self):
super().__init__()
# For cache
self.mapping_to_detectron = None
self.orphans_in_detectron = None
# Backbone for feature extraction
self.Conv_Body = get_func(cfg.MODEL.CONV_BODY)()
# Layers for left-right view
# Added by zhangfandong
# LR_VIEW and GIF should not be on at the same time
assert (not cfg.MODEL.LR_VIEW_ON) or (not cfg.MODEL.GIF_ON) or (
not cfg.MODEL.LRASY_MAHA_ON)
if cfg.MODEL.LR_VIEW_ON:
#assert cfg.FPN.FPN_ON is False
self.lr_view_net = lrview_net_body.get_lrv_net(
self.Conv_Body.dim_out * 2, self.Conv_Body.dim_out,
cfg.LR_VIEW.FUSION_NET)
elif cfg.MODEL.LRASY_MAHA_ON:
self.lrasy_maha_net = lrview_net_body.get_lrvasymaha_net(
self.Conv_Body.dim_out, self.Conv_Body.dim_out,
cfg.LR_VIEW.FUSION_NET)
elif cfg.MODEL.GIF_ON:
self.gif_net = gif_net_body.get_gif_net(self.Conv_Body.dim_out)
# Region Proposal Network
if cfg.RPN.RPN_ON:
self.RPN = rpn_heads.generic_rpn_outputs(
self.Conv_Body.dim_out, self.Conv_Body.spatial_scale)
if cfg.FPN.FPN_ON:
# Only supports case when RPN and ROI min levels are the same
#assert cfg.FPN.RPN_MIN_LEVEL == cfg.FPN.ROI_MIN_LEVEL
# RPN max level can be >= to ROI max level
assert cfg.FPN.RPN_MAX_LEVEL >= cfg.FPN.ROI_MAX_LEVEL
# FPN RPN max level might be > FPN ROI max level in which case we
# need to discard some leading conv blobs (blobs are ordered from
# max/coarsest level to min/finest level)
self.num_roi_levels = cfg.FPN.ROI_MAX_LEVEL - cfg.FPN.ROI_MIN_LEVEL + 1
# Retain only the spatial scales that will be used for RoI heads. `Conv_Body.spatial_scale`
# may include extra scales that are used for RPN proposals, but not for RoI heads.
self.Conv_Body.spatial_scale = self.Conv_Body.spatial_scale[
-self.num_roi_levels:]
# BBOX Branch
if not cfg.MODEL.RPN_ONLY:
self.Box_Head = get_func(cfg.FAST_RCNN.ROI_BOX_HEAD)(
self.RPN.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
self.Box_Outs = fast_rcnn_heads.fast_rcnn_outputs(
self.Box_Head.dim_out)
if cfg.LESION.POSITION_RCNN:
self.Position_RCNN = position_outputs(self.Box_Head.dim_out)
# Position Branch
if cfg.LESION.USE_POSITION:
#self.Position_Head = position_Xconv1fc_gn_head(2048,1024,2)
self.Position_Head = position_Xconv1fc_gn_head(2048, 1024, 3)
self.Position_Cls_Outs = position_cls_outputs(
self.Position_Head.dim_out)
self.Position_Reg_Outs = position_reg_outputs(
self.Position_Head.dim_out)
# Mask Branch
if cfg.MODEL.MASK_ON:
self.Mask_Head = get_func(cfg.MRCNN.ROI_MASK_HEAD)(
self.RPN.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
if getattr(self.Mask_Head, 'SHARE_RES5', False):
self.Mask_Head.share_res5_module(self.Box_Head.res5)
self.Mask_Outs = mask_rcnn_heads.mask_rcnn_outputs(
self.Mask_Head.dim_out)
# Keypoints Branch
if cfg.MODEL.KEYPOINTS_ON:
self.Keypoint_Head = get_func(cfg.KRCNN.ROI_KEYPOINTS_HEAD)(
self.RPN.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
if getattr(self.Keypoint_Head, 'SHARE_RES5', False):
self.Keypoint_Head.share_res5_module(self.Box_Head.res5)
self.Keypoint_Outs = keypoint_rcnn_heads.keypoint_outputs(
self.Keypoint_Head.dim_out)
self._init_modules()
def __init__(self):
super().__init__()
# Stop forwarding from Conv-GRU module.
self.stop_after_hidden_states = False
# For cache
self.mapping_to_detectron = None
self.orphans_in_detectron = None
self.update_hidden_states = True
# Backbone for feature extraction
self.Conv_Body = get_func(cfg.MODEL.CONV_BODY)()
assert (cfg.FPN.FPN_ON)
#insert temporal module for video object segmentation.
#As we use resNet, dims for 5 levels are hard coded.
#fpn_dims=(2048, 1024, 512, 256, 64)
fpn_dims = [cfg.FPN.DIM] * 5
h_channels = cfg.CONVGRU.HIDDEN_STATE_CHANNELS
self.ConvGRUs = nn.ModuleList()
for i in range(len(fpn_dims)):
self.ConvGRUs.append(
ConvGRUCell2d(fpn_dims[i],
h_channels[i],
kernel_size=3,
stride=1,
dilation=1,
groups=1,
use_GN=True,
GN_groups=32))
self.flow_features = None
if cfg.MODEL.USE_DELTA_FLOW:
self.flow_features = [None] * 5
self.Conv_Delta_Flows_Features = nn.ModuleList()
#TODO add initialization.
for i in range(len(fpn_dims)):
self.Conv_Delta_Flows_Features.append(
nn.Sequential(
nn.Conv2d(fpn_dims[i],
2,
kernel_size=3,
stride=1,
padding=1,
bias=False), nn.Tanh()))
self.hidden_states = [None, None, None, None, None]
# Region Proposal Network
if cfg.RPN.RPN_ON:
self.RPN = rpn_heads.generic_rpn_outputs(
self.Conv_Body.dim_out, self.Conv_Body.spatial_scale)
if cfg.FPN.FPN_ON:
# Only supports case when RPN and ROI min levels are the same
assert cfg.FPN.RPN_MIN_LEVEL == cfg.FPN.ROI_MIN_LEVEL
# RPN max level can be >= to ROI max level
assert cfg.FPN.RPN_MAX_LEVEL >= cfg.FPN.ROI_MAX_LEVEL
# FPN RPN max level might be > FPN ROI max level in which case we
# need to discard some leading conv blobs (blobs are ordered from
# max/coarsest level to min/finest level)
self.num_roi_levels = cfg.FPN.ROI_MAX_LEVEL - cfg.FPN.ROI_MIN_LEVEL + 1
# Retain only the spatial scales that will be used for RoI heads. `Conv_Body.spatial_scale`
# may include extra scales that are used for RPN proposals, but not for RoI heads.
self.Conv_Body.spatial_scale = self.Conv_Body.spatial_scale[
-self.num_roi_levels:]
# BBOX Branch
if not cfg.MODEL.RPN_ONLY:
self.Box_Head = get_func(cfg.FAST_RCNN.ROI_BOX_HEAD)(
self.RPN.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
if not cfg.MODEL.IDENTITY_TRAINING:
self.Box_Outs = fast_rcnn_heads.fast_rcnn_outputs(
self.Box_Head.dim_out)
else:
self.Box_Outs = fast_rcnn_heads.fast_rcnn_outputs(
self.Box_Head.dim_out)
# Mask Branch
if cfg.MODEL.MASK_ON:
self.Mask_Head = get_func(cfg.MRCNN.ROI_MASK_HEAD)(
self.RPN.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
if getattr(self.Mask_Head, 'SHARE_RES5', False):
self.Mask_Head.share_res5_module(self.Box_Head.res5)
self.Mask_Outs = mask_rcnn_heads.mask_rcnn_outputs(
self.Mask_Head.dim_out, cfg.MODEL.NUM_CLASSES)
# Keypoints Branch
if cfg.MODEL.KEYPOINTS_ON:
self.Keypoint_Head = get_func(cfg.KRCNN.ROI_KEYPOINTS_HEAD)(
self.RPN.dim_out, self.roi_feature_transform,
self.Conv_Body.spatial_scale)
if getattr(self.Keypoint_Head, 'SHARE_RES5', False):
self.Keypoint_Head.share_res5_module(self.Box_Head.res5)
self.Keypoint_Outs = keypoint_rcnn_heads.keypoint_outputs(
self.Keypoint_Head.dim_out)
if cfg.CONVGRU.DYNAMIC_MODEL:
self.FlowAligns = nn.ModuleList()
self.fpn_scales = [1. / 64., 1. / 32., 1. / 16., 1. / 8., 1. / 4.]
for i in range(len(self.fpn_scales)):
self.FlowAligns.append(FlowAlign(self.fpn_scales[i]))
self._init_modules()
def __init__(self):
super().__init__()
# For cache
self.mapping_to_detectron = None
self.orphans_in_detectron = None
# Backbone for feature extraction
self.Conv_Body = get_func(cfg.MODEL.CONV_BODY)()
# Region Proposal Network
if cfg.RPN.RPN_ON:
self.RPN = rpn_heads.generic_rpn_outputs(
self.Conv_Body.dim_out, self.Conv_Body.spatial_scale)
if cfg.FPN.FPN_ON:
# Only supports case when RPN and ROI min levels are the same
assert cfg.FPN.RPN_MIN_LEVEL == cfg.FPN.ROI_MIN_LEVEL
# RPN max level can be >= to ROI max level
assert cfg.FPN.RPN_MAX_LEVEL >= cfg.FPN.ROI_MAX_LEVEL
# FPN RPN max level might be > FPN ROI max level in which case we
# need to discard some leading conv blobs (blobs are ordered from
# max/coarsest level to min/finest level)
self.num_roi_levels = cfg.FPN.ROI_MAX_LEVEL - cfg.FPN.ROI_MIN_LEVEL + 1
# Retain only the spatial scales that will be used for RoI heads. `Conv_Body.spatial_scale`
# may include extra scales that are used for RPN proposals, but not for RoI heads.
self.Conv_Body.spatial_scale = self.Conv_Body.spatial_scale[-self.num_roi_levels:]
# BBOX Branch
if not cfg.MODEL.RPN_ONLY:
self.Box_Head = get_func(cfg.FAST_RCNN.ROI_BOX_HEAD)(
self.RPN.dim_out, self.roi_feature_transform, self.Conv_Body.spatial_scale)
self.Box_Outs = fast_rcnn_heads.fast_rcnn_outputs(
self.Box_Head.dim_out)
# Mask Branch
if cfg.MODEL.MASK_ON:
self.Mask_Head = get_func(cfg.MRCNN.ROI_MASK_HEAD)(
self.RPN.dim_out, self.roi_feature_transform, self.Conv_Body.spatial_scale)
if getattr(self.Mask_Head, 'SHARE_RES5', False):
self.Mask_Head.share_res5_module(self.Box_Head.res5)
self.Mask_Outs = mask_rcnn_heads.mask_rcnn_outputs(self.Mask_Head.dim_out)
# Keypoints Branch
if cfg.MODEL.KEYPOINTS_ON:
self.Keypoint_Head = get_func(cfg.KRCNN.ROI_KEYPOINTS_HEAD)(
self.RPN.dim_out, self.roi_feature_transform, self.Conv_Body.spatial_scale)
if getattr(self.Keypoint_Head, 'SHARE_RES5', False):
self.Keypoint_Head.share_res5_module(self.Box_Head.res5)
self.Keypoint_Outs = keypoint_rcnn_heads.keypoint_outputs(self.Keypoint_Head.dim_out)
self._init_modules()
# Domain Discriminator Branch
if cfg.TRAIN.DOMAIN_ADAPT_IM:
self.DiscriminatorImage_Head = adversarial_heads.domain_discriminator_im(
grl_scaler=cfg.TRAIN.GRL_SCALER)
if cfg.TRAIN.DOMAIN_ADAPT_ROI:
self.DiscriminatorRoi_Head = adversarial_heads.domain_discriminator_roi(
grl_scaler=cfg.TRAIN.GRL_SCALER)
# Learned attention for distillation lambda Branch
if cfg.TRAIN.DISTILL_ATTN:
self.AttentionRoi_Head = fast_rcnn_heads.distill_attention_head()
