def __init__(self, config, in_channels):
super(IMPPredictor, self).__init__()
self.num_obj_cls = config.MODEL.ROI_BOX_HEAD.NUM_CLASSES
self.num_rel_cls = config.MODEL.ROI_RELATION_HEAD.NUM_CLASSES
self.use_bias = False
assert in_channels is not None
self.context_layer = IMPContext(config, self.num_obj_cls,
self.num_rel_cls, in_channels)
# post decoding
self.hidden_dim = config.MODEL.ROI_RELATION_HEAD.CONTEXT_HIDDEN_DIM
self.pooling_dim = config.MODEL.ROI_RELATION_HEAD.CONTEXT_POOLING_DIM
if self.pooling_dim != config.MODEL.ROI_BOX_HEAD.MLP_HEAD_DIM:
self.union_single_not_match = True
self.up_dim = nn.Linear(config.MODEL.ROI_BOX_HEAD.MLP_HEAD_DIM,
self.pooling_dim)
layer_init(self.up_dim, xavier=True)
else:
self.union_single_not_match = False
# freq
if self.use_bias:
statistics = get_dataset_statistics(config)
self.freq_bias = FrequencyBias(config, statistics)
def __init__(self, config, in_channels):
super(MotifPredictor, self).__init__()
self.attribute_on = config.MODEL.ATTRIBUTE_ON
self.num_obj_cls = config.MODEL.ROI_BOX_HEAD.NUM_CLASSES
self.num_att_cls = config.MODEL.ROI_ATTRIBUTE_HEAD.NUM_ATTRIBUTES
self.num_rel_cls = config.MODEL.ROI_RELATION_HEAD.NUM_CLASSES
assert in_channels is not None
num_inputs = in_channels
self.use_vision = config.MODEL.ROI_RELATION_HEAD.PREDICT_USE_VISION
self.use_bias = config.MODEL.ROI_RELATION_HEAD.PREDICT_USE_BIAS
# load class dict
statistics = get_dataset_statistics(config)
obj_classes, rel_classes, att_classes = statistics[
'obj_classes'], statistics['rel_classes'], statistics[
'att_classes']
assert self.num_obj_cls == len(obj_classes)
assert self.num_att_cls == len(att_classes)
assert self.num_rel_cls == len(rel_classes)
# init contextual lstm encoding
if self.attribute_on:
self.context_layer = AttributeLSTMContext(config, obj_classes,
att_classes, rel_classes,
in_channels)
else:
self.context_layer = LSTMContext(config, obj_classes, rel_classes,
in_channels)
# post decoding
self.hidden_dim = config.MODEL.ROI_RELATION_HEAD.CONTEXT_HIDDEN_DIM
self.pooling_dim = config.MODEL.ROI_RELATION_HEAD.CONTEXT_POOLING_DIM
self.post_emb = nn.Linear(self.hidden_dim, self.hidden_dim * 2)
self.post_cat = nn.Linear(self.hidden_dim * 2, self.pooling_dim)
self.rel_compress = nn.Linear(self.pooling_dim,
self.num_rel_cls,
bias=True)
# initialize layer parameters
layer_init(self.post_emb,
10.0 * (1.0 / self.hidden_dim)**0.5,
normal=True)
layer_init(self.post_cat, xavier=True)
layer_init(self.rel_compress, xavier=True)
if self.pooling_dim != config.MODEL.ROI_BOX_HEAD.MLP_HEAD_DIM:
self.union_single_not_match = True
self.up_dim = nn.Linear(config.MODEL.ROI_BOX_HEAD.MLP_HEAD_DIM,
self.pooling_dim)
layer_init(self.up_dim, xavier=True)
else:
self.union_single_not_match = False
if self.use_bias:
# convey statistics into FrequencyBias to avoid loading again
self.freq_bias = FrequencyBias(config, statistics)
def __init__(self, config, in_channels):
super(VCTreePredictor, self).__init__()
self.attribute_on = config.MODEL.ATTRIBUTE_ON
self.num_obj_cls = config.MODEL.ROI_BOX_HEAD.NUM_CLASSES
self.num_att_cls = config.MODEL.ROI_ATTRIBUTE_HEAD.NUM_ATTRIBUTES
self.num_rel_cls = config.MODEL.ROI_RELATION_HEAD.NUM_CLASSES
assert in_channels is not None
num_inputs = in_channels
# load class dict
statistics = get_dataset_statistics(config)
obj_classes, rel_classes, att_classes = statistics[
'obj_classes'], statistics['rel_classes'], statistics[
'att_classes']
assert self.num_obj_cls == len(obj_classes)
assert self.num_att_cls == len(att_classes)
assert self.num_rel_cls == len(rel_classes)
# init contextual lstm encoding
self.context_layer = VCTreeLSTMContext(config, obj_classes,
rel_classes, statistics,
in_channels)
# post decoding
self.hidden_dim = config.MODEL.ROI_RELATION_HEAD.CONTEXT_HIDDEN_DIM
self.pooling_dim = config.MODEL.ROI_RELATION_HEAD.CONTEXT_POOLING_DIM
self.post_emb = nn.Linear(self.hidden_dim, self.hidden_dim * 2)
self.post_cat = nn.Linear(self.hidden_dim * 2, self.pooling_dim)
# learned-mixin
#self.uni_gate = nn.Linear(self.pooling_dim, self.num_rel_cls)
#self.frq_gate = nn.Linear(self.pooling_dim, self.num_rel_cls)
self.ctx_compress = nn.Linear(self.pooling_dim, self.num_rel_cls)
#self.uni_compress = nn.Linear(self.pooling_dim, self.num_rel_cls)
#layer_init(self.uni_gate, xavier=True)
#layer_init(self.frq_gate, xavier=True)
layer_init(self.ctx_compress, xavier=True)
#layer_init(self.uni_compress, xavier=True)
# initialize layer parameters
layer_init(self.post_emb,
10.0 * (1.0 / self.hidden_dim)**0.5,
normal=True)
layer_init(self.post_cat, xavier=True)
if self.pooling_dim != config.MODEL.ROI_BOX_HEAD.MLP_HEAD_DIM:
self.union_single_not_match = True
self.up_dim = nn.Linear(config.MODEL.ROI_BOX_HEAD.MLP_HEAD_DIM,
self.pooling_dim)
layer_init(self.up_dim, xavier=True)
else:
self.union_single_not_match = False
self.freq_bias = FrequencyBias(config, statistics)
def __init__(self, config, in_channels):
super(CausalAnalysisPredictor, self).__init__()
self.cfg = config
self.attribute_on = config.MODEL.ATTRIBUTE_ON
self.spatial_for_vision = config.MODEL.ROI_RELATION_HEAD.CAUSAL.SPATIAL_FOR_VISION
self.num_obj_cls = config.MODEL.ROI_BOX_HEAD.NUM_CLASSES
self.num_rel_cls = config.MODEL.ROI_RELATION_HEAD.NUM_CLASSES
self.fusion_type = config.MODEL.ROI_RELATION_HEAD.CAUSAL.FUSION_TYPE
self.separate_spatial = config.MODEL.ROI_RELATION_HEAD.CAUSAL.SEPARATE_SPATIAL
self.use_vtranse = config.MODEL.ROI_RELATION_HEAD.CAUSAL.CONTEXT_LAYER == "vtranse"
self.effect_type = config.MODEL.ROI_RELATION_HEAD.CAUSAL.EFFECT_TYPE
assert in_channels is not None
num_inputs = in_channels
# load class dict
statistics = get_dataset_statistics(config)
obj_classes, rel_classes = statistics['obj_classes'], statistics[
'rel_classes']
assert self.num_obj_cls == len(obj_classes)
assert self.num_rel_cls == len(rel_classes)
# init contextual lstm encoding
if config.MODEL.ROI_RELATION_HEAD.CAUSAL.CONTEXT_LAYER == "motifs":
self.context_layer = LSTMContext(config, obj_classes, rel_classes,
in_channels)
elif config.MODEL.ROI_RELATION_HEAD.CAUSAL.CONTEXT_LAYER == "vctree":
self.context_layer = VCTreeLSTMContext(config, obj_classes,
rel_classes, statistics,
in_channels)
elif config.MODEL.ROI_RELATION_HEAD.CAUSAL.CONTEXT_LAYER == "vtranse":
self.context_layer = VTransEFeature(config, obj_classes,
rel_classes, in_channels)
else:
print('ERROR: Invalid Context Layer')
# post decoding
self.hidden_dim = config.MODEL.ROI_RELATION_HEAD.CONTEXT_HIDDEN_DIM
self.pooling_dim = config.MODEL.ROI_RELATION_HEAD.CONTEXT_POOLING_DIM
if self.use_vtranse:
self.edge_dim = self.pooling_dim
self.post_emb = nn.Linear(self.hidden_dim, self.pooling_dim * 2)
self.ctx_compress = nn.Linear(self.pooling_dim,
self.num_rel_cls,
bias=False)
else:
self.edge_dim = self.hidden_dim
self.post_emb = nn.Linear(self.hidden_dim, self.hidden_dim * 2)
self.post_cat = nn.Sequential(*[
nn.Linear(self.hidden_dim * 2, self.pooling_dim),
nn.ReLU(inplace=True),
])
self.ctx_compress = nn.Linear(self.pooling_dim, self.num_rel_cls)
self.vis_compress = nn.Linear(self.pooling_dim, self.num_rel_cls)
if self.fusion_type == 'gate':
self.ctx_gate_fc = nn.Linear(self.pooling_dim, self.num_rel_cls)
layer_init(self.ctx_gate_fc, xavier=True)
# initialize layer parameters
layer_init(self.post_emb,
10.0 * (1.0 / self.hidden_dim)**0.5,
normal=True)
if not self.use_vtranse:
layer_init(self.post_cat[0], xavier=True)
layer_init(self.ctx_compress, xavier=True)
layer_init(self.vis_compress, xavier=True)
assert self.pooling_dim == config.MODEL.ROI_BOX_HEAD.MLP_HEAD_DIM
# convey statistics into FrequencyBias to avoid loading again
self.freq_bias = FrequencyBias(config, statistics)
# add spatial emb for visual feature
if self.spatial_for_vision:
self.spt_emb = nn.Sequential(*[
nn.Linear(32, self.hidden_dim),
nn.ReLU(inplace=True),
nn.Linear(self.hidden_dim, self.pooling_dim),
nn.ReLU(inplace=True)
])
layer_init(self.spt_emb[0], xavier=True)
layer_init(self.spt_emb[2], xavier=True)
self.label_smooth_loss = Label_Smoothing_Regression(e=1.0)
# untreated average features
self.effect_analysis = config.MODEL.ROI_RELATION_HEAD.CAUSAL.EFFECT_ANALYSIS
self.average_ratio = 0.0005
self.register_buffer("untreated_spt", torch.zeros(32))
self.register_buffer("untreated_conv_spt",
torch.zeros(self.pooling_dim))
self.register_buffer("avg_post_ctx", torch.zeros(self.pooling_dim))
self.register_buffer("untreated_feat", torch.zeros(self.pooling_dim))