def get_classifier(self, pooling_size=7): """ Get the classifier network corresponding to the specified depth feature extractor. The classifier is used as a relation head or classification head. :param pooling_size: RoI pooling size """ # -- AlexNet modified classifier -- if self.depth_model == 'alexnet': classifier = nn.Sequential( nn.Dropout(), # -- Changed the input size ( from [6,6] to [7,7]) nn.Linear(256 * pooling_size**2, 4096), nn.ReLU(inplace=True), nn.Dropout(), nn.Linear(4096, 4096), nn.ReLU(inplace=True) # -- Ignore the final layer # nn.Linear(4096, num_classes) ) self.init_weights(classifier) return classifier # -- ResNet 18 modified classifier -- elif self.depth_model == 'resnet18': return nn.Sequential( resnet18_l4(relu_end=False, pretrained=False), nn.AvgPool2d(pooling_size), Flattener(), ) # -- ResNet 50 modified classifier -- elif self.depth_model == 'resnet50': return nn.Sequential( resnet50_l4(relu_end=False, pretrained=False), nn.AvgPool2d(pooling_size), Flattener(), ) # -- VGG 16 classifier part -- elif self.depth_model == 'vgg': return load_vgg(pretrained=False).classifier # -- SqueezeNet 1.1 modified classifier -- elif self.depth_model == 'sqznet': classifier = nn.Sequential(nn.Dropout(), nn.Conv2d(512, 1024, kernel_size=1), nn.ReLU(inplace=True), nn.AvgPool2d(pooling_size), Flattener()) self.init_weights(classifier) return classifier
def __init__(self, classes, num_rels, mode='sgdet', embed_dim=200, pooling_dim=4096, use_bias=True): super(EndCell, self).__init__() self.classes = classes self.num_rels = num_rels assert mode in MODES self.embed_dim = embed_dim self.pooling_dim = pooling_dim self.use_bias = use_bias self.mode = mode self.ort_embedding = torch.autograd.Variable( get_ort_embeds(self.num_classes, self.embed_dim).cuda()) self.context = LC(classes=self.classes, mode=self.mode, embed_dim=self.embed_dim, obj_dim=self.pooling_dim) self.union_boxes = UnionBoxesAndFeats(pooling_size=7, stride=16, dim=512) self.pooling_size = 7 roi_fmap = [ Flattener(), load_vgg(use_dropout=False, use_relu=False, use_linear=pooling_dim == 4096, pretrained=False).classifier, ] if pooling_dim != 4096: roi_fmap.append(nn.Linear(4096, pooling_dim)) self.roi_fmap = nn.Sequential(*roi_fmap) self.roi_fmap_obj = load_vgg(pretrained=False).classifier self.post_lstm = nn.Linear(self.pooling_dim + self.embed_dim + 5, self.pooling_dim * 2) # Initialize to sqrt(1/2n) so that the outputs all have mean 0 and variance 1. # (Half contribution comes from LSTM, half from embedding. # In practice the pre-lstm stuff tends to have stdev 0.1 so I multiplied this by 10. self.post_lstm.weight.data.normal_( 0, 10.0 * math.sqrt(1.0 / self.pooling_dim)) self.post_lstm.bias.data.zero_() self.post_emb = nn.Linear(self.pooling_dim + self.embed_dim + 5, self.pooling_dim * 2) self.rel_compress = nn.Linear(self.pooling_dim, self.num_rels, bias=True) self.rel_compress.weight = torch.nn.init.xavier_normal( self.rel_compress.weight, gain=1.0) if self.use_bias: self.freq_bias = FrequencyBias()
def __init__(self, num_classes, ctx_dim=512): super(GlobalContextEncoding, self).__init__() self.glb_avg_pool = nn.Sequential( nn.AdaptiveAvgPool2d(output_size=(1, 1)), Flattener(), ) self.multi_score_fc = nn.Linear(ctx_dim, num_classes)
def __init__(self, feat_dim, h_dim, pass_root, is_pass_embed=False, embed_layer=None, embed_out_layer=None, saliency=False): super(HrTreeLSTM_Foreward, self).__init__() self.feat_dim = feat_dim self.h_dim = h_dim self.pass_root = pass_root self.is_pass_embed = is_pass_embed self.embed_layer = embed_layer self.embed_out_layer = embed_out_layer self.saliency = saliency self.pooling_size = 7 if self.is_pass_embed: assert self.embed_layer is not None self.p_embed_dim = self.embed_layer.weight.data.shape[1] self.p_embed = nn.Linear(self.p_embed_dim, self.p_embed_dim) block_orthogonal(self.p_embed.weight.data, [self.p_embed_dim, self.p_embed_dim]) self.p_embed.bias.data.fill_(0.0) feat_forward_dim = self.feat_dim + self.p_embed_dim else: feat_forward_dim = self.feat_dim self.ioffux = nn.Linear(feat_forward_dim, 4 * self.h_dim) self.ioffuh = nn.Linear(self.h_dim, 4 * self.h_dim) self.px = nn.Linear(feat_forward_dim, self.h_dim) self.forget_x = nn.Linear(feat_forward_dim, self.h_dim) self.forget_h = nn.Linear(self.h_dim, self.h_dim) # init parameter block_orthogonal(self.px.weight.data, [self.h_dim, feat_forward_dim]) block_orthogonal(self.ioffux.weight.data, [self.h_dim, feat_forward_dim]) block_orthogonal(self.ioffuh.weight.data, [self.h_dim, self.h_dim]) block_orthogonal(self.forget_x.weight.data, [self.h_dim, feat_forward_dim]) block_orthogonal(self.forget_h.weight.data, [self.h_dim, self.h_dim]) self.px.bias.data.fill_(0.0) self.ioffux.bias.data.fill_(0.0) self.ioffuh.bias.data.fill_(0.0) self.forget_x.bias.data.fill_(0.0) # Initialize forget gate biases to 1.0 as per An Empirical # Exploration of Recurrent Network Architectures, (Jozefowicz, 2015). self.forget_h.bias.data.fill_(1.0) if self.saliency: sal_fmap = [Flattener(), nn.Linear(self.pooling_size * self.pooling_size + 3, 1)] self.sal_fmap = nn.Linear(self.pooling_size * self.pooling_size + 3, 1)
def __init__(self, classes, rel_classes, mode='sgdet', num_gpus=1, require_overlap_det=True, pooling_dim=4096, use_resnet=False, thresh=0.01, use_proposals=False, use_ggnn_obj=False, ggnn_obj_time_step_num=3, ggnn_obj_hidden_dim=512, ggnn_obj_output_dim=512, use_ggnn_rel=False, ggnn_rel_time_step_num=3, ggnn_rel_hidden_dim=512, ggnn_rel_output_dim=512, use_obj_knowledge=True, use_rel_knowledge=True, obj_knowledge='', rel_knowledge=''): """ :param classes: Object classes :param rel_classes: Relationship classes. None if were not using rel mode :param mode: (sgcls, predcls, or sgdet) :param num_gpus: how many GPUS 2 use :param require_overlap_det: Whether two objects must intersect """ super(KERN, self).__init__() self.classes = classes self.rel_classes = rel_classes self.num_gpus = num_gpus assert mode in MODES self.mode = mode self.pooling_size = 7 self.obj_dim = 2048 if use_resnet else 4096 self.rel_dim = self.obj_dim self.pooling_dim = pooling_dim self.use_ggnn_obj = use_ggnn_obj self.use_ggnn_rel = use_ggnn_rel self.require_overlap = require_overlap_det and self.mode == 'sgdet' self.detector = ObjectDetector( classes=classes, mode=('proposals' if use_proposals else 'refinerels') if mode == 'sgdet' else 'gtbox', use_resnet=use_resnet, thresh=thresh, max_per_img=64) self.union_boxes = UnionBoxesAndFeats(pooling_size=self.pooling_size, stride=16, dim=1024 if use_resnet else 512) if use_resnet: self.roi_fmap = nn.Sequential( resnet_l4(relu_end=False), nn.AvgPool2d(self.pooling_size), Flattener(), ) else: roi_fmap = [ Flattener(), load_vgg(use_dropout=False, use_relu=False, use_linear=pooling_dim == 4096, pretrained=False).classifier, ] if pooling_dim != 4096: roi_fmap.append(nn.Linear(4096, pooling_dim)) self.roi_fmap = nn.Sequential(*roi_fmap) self.roi_fmap_obj = load_vgg(pretrained=False).classifier if self.use_ggnn_obj: self.ggnn_obj_reason = GGNNObjReason( mode=self.mode, num_obj_cls=len(self.classes), obj_dim=self.obj_dim, time_step_num=ggnn_obj_time_step_num, hidden_dim=ggnn_obj_hidden_dim, output_dim=ggnn_obj_output_dim, use_knowledge=use_obj_knowledge, knowledge_matrix=obj_knowledge) if self.use_ggnn_rel: self.ggnn_rel_reason = GGNNRelReason( mode=self.mode, num_obj_cls=len(self.classes), num_rel_cls=len(rel_classes), obj_dim=self.obj_dim, rel_dim=self.rel_dim, time_step_num=ggnn_rel_time_step_num, hidden_dim=ggnn_rel_hidden_dim, output_dim=ggnn_obj_output_dim, use_knowledge=use_rel_knowledge, knowledge_matrix=rel_knowledge) else: self.vr_fc_cls = VRFC(self.mode, self.rel_dim, len(self.classes), len(self.rel_classes))
def __init__(self, classes, rel_classes, graph_path, emb_path, mode='sgdet', num_gpus=1, require_overlap_det=True, pooling_dim=4096, use_resnet=False, thresh=0.01, use_proposals=False, ggnn_rel_time_step_num=3, ggnn_rel_hidden_dim=512, ggnn_rel_output_dim=512, use_knowledge=True, use_embedding=True, refine_obj_cls=False, rel_counts_path=None, class_volume=1.0, top_k_to_keep=5, normalize_messages=True): """ :param classes: Object classes :param rel_classes: Relationship classes. None if were not using rel mode :param mode: (sgcls, predcls, or sgdet) :param num_gpus: how many GPUS 2 use :param require_overlap_det: Whether two objects must intersect """ super(KERN, self).__init__() self.classes = classes self.rel_classes = rel_classes self.num_gpus = num_gpus assert mode in MODES self.mode = mode self.pooling_size = 7 self.obj_dim = 2048 if use_resnet else 4096 self.rel_dim = self.obj_dim self.pooling_dim = pooling_dim self.require_overlap = require_overlap_det and self.mode == 'sgdet' self.detector = ObjectDetector( classes=classes, mode=('proposals' if use_proposals else 'refinerels') if mode == 'sgdet' else 'gtbox', use_resnet=use_resnet, thresh=thresh, max_per_img=64 ) self.union_boxes = UnionBoxesAndFeats(pooling_size=self.pooling_size, stride=16, dim=1024 if use_resnet else 512) if use_resnet: self.roi_fmap = nn.Sequential( resnet_l4(relu_end=False), nn.AvgPool2d(self.pooling_size), Flattener(), ) else: roi_fmap = [ Flattener(), load_vgg(use_dropout=False, use_relu=False, use_linear=pooling_dim == 4096, pretrained=False).classifier, ] if pooling_dim != 4096: roi_fmap.append(nn.Linear(4096, pooling_dim)) self.roi_fmap = nn.Sequential(*roi_fmap) self.roi_fmap_obj = load_vgg(pretrained=False).classifier self.ggnn_rel_reason = GGNNRelReason(mode=self.mode, num_obj_cls=len(self.classes), num_rel_cls=len(rel_classes), obj_dim=self.obj_dim, rel_dim=self.rel_dim, time_step_num=ggnn_rel_time_step_num, hidden_dim=ggnn_rel_hidden_dim, output_dim=ggnn_rel_output_dim, emb_path=emb_path, graph_path=graph_path, refine_obj_cls=refine_obj_cls, use_knowledge=use_knowledge, use_embedding=use_embedding, top_k_to_keep=top_k_to_keep, normalize_messages=normalize_messages ) if rel_counts_path is not None: with open(rel_counts_path, 'rb') as fin: rel_counts = pickle.load(fin) beta = (class_volume - 1.0) / class_volume self.rel_class_weights = (1.0 - beta) / (1 - (beta ** rel_counts)) self.rel_class_weights *= float(self.num_rels) / np.sum(self.rel_class_weights) else: self.rel_class_weights = np.ones((self.num_rels,)) self.rel_class_weights = Variable(torch.from_numpy(self.rel_class_weights).float().cuda(), requires_grad=False)
def __init__(self, classes, rel_classes, mode='sgdet', num_gpus=1, use_vision=True, require_overlap_det=True, embed_dim=200, hidden_dim=256, pooling_dim=2048, nl_obj=1, nl_edge=2, use_resnet=False, order='confidence', thresh=0.01, use_proposals=False, pass_in_obj_feats_to_decoder=True, gnn=True, reachability=False, pass_in_obj_feats_to_edge=True, rec_dropout=0.0, use_bias=True, use_tanh=True, limit_vision=True): """ :param classes: Object classes :param rel_classes: Relationship classes. None if were not using rel mode :param mode: (sgcls, predcls, or sgdet) :param num_gpus: how many GPUS 2 use :param use_vision: Whether to use vision in the final product :param require_overlap_det: Whether two objects must intersect :param embed_dim: Dimension for all embeddings :param hidden_dim: LSTM hidden size :param obj_dim: """ super(RelModel, self).__init__() self.classes = classes self.rel_classes = rel_classes self.num_gpus = num_gpus assert mode in MODES self.mode = mode self.reachability = reachability self.gnn = gnn self.pooling_size = 7 self.embed_dim = embed_dim self.hidden_dim = hidden_dim self.obj_dim = 2048 if use_resnet else 4096 self.pooling_dim = pooling_dim self.use_bias = use_bias self.use_vision = use_vision self.use_tanh = use_tanh self.limit_vision = limit_vision self.require_overlap = require_overlap_det and self.mode == 'sgdet' self.global_embedding = EmbeddingImagenet(4096) self.global_logist = nn.Linear(4096, 151, bias=True) # CosineLinear(4096,150)# self.global_logist.weight = torch.nn.init.xavier_normal( self.global_logist.weight, gain=1.0) self.disc_center = DiscCentroidsLoss(self.num_rels, self.pooling_dim + 256) self.meta_classify = MetaEmbedding_Classifier( feat_dim=self.pooling_dim + 256, num_classes=self.num_rels) # self.global_rel_logist = nn.Linear(4096, 50 , bias=True) # self.global_rel_logist.weight = torch.nn.init.xavier_normal(self.global_rel_logist.weight, gain=1.0) # self.global_logist = CosineLinear(4096,150) self.global_sub_additive = nn.Linear(4096, 1, bias=True) self.global_obj_additive = nn.Linear(4096, 1, bias=True) self.detector = ObjectDetector( classes=classes, mode=('proposals' if use_proposals else 'refinerels') if mode == 'sgdet' else 'gtbox', use_resnet=use_resnet, thresh=thresh, max_per_img=64, ) self.context = LinearizedContext( self.classes, self.rel_classes, mode=self.mode, embed_dim=self.embed_dim, hidden_dim=self.hidden_dim, obj_dim=self.obj_dim, nl_obj=nl_obj, nl_edge=nl_edge, dropout_rate=rec_dropout, order=order, pass_in_obj_feats_to_decoder=pass_in_obj_feats_to_decoder, pass_in_obj_feats_to_edge=pass_in_obj_feats_to_edge) # Image Feats (You'll have to disable if you want to turn off the features from here) self.union_boxes = UnionBoxesAndFeats(pooling_size=self.pooling_size, stride=16, dim=1024 if use_resnet else 512) if use_resnet: self.roi_fmap = nn.Sequential( resnet_l4(relu_end=False), nn.AvgPool2d(self.pooling_size), Flattener(), ) else: roi_fmap = [ Flattener(), load_vgg(use_dropout=False, use_relu=False, use_linear=pooling_dim == 4096, pretrained=False).classifier, ] if pooling_dim != 4096: roi_fmap.append(nn.Linear(4096, pooling_dim)) self.roi_fmap = nn.Sequential(*roi_fmap) self.roi_fmap_obj = load_vgg(pretrained=False).classifier ################################### self.post_lstm = nn.Linear(self.hidden_dim, self.pooling_dim * 2) self.edge_coordinate_embedding = nn.Sequential(*[ nn.BatchNorm1d(5, momentum=BATCHNORM_MOMENTUM / 10.0), nn.Linear(5, 256), nn.ReLU(inplace=True), nn.Dropout(0.1), ]) # Initialize to sqrt(1/2n) so that the outputs all have mean 0 and variance 1. # (Half contribution comes from LSTM, half from embedding. # In practice the pre-lstm stuff tends to have stdev 0.1 so I multiplied this by 10. self.post_lstm.weight.data.normal_( 0, 10.0 * math.sqrt(1.0 / self.hidden_dim)) self.post_lstm.bias.data.zero_() if nl_edge == 0: self.post_emb = nn.Embedding(self.num_classes, self.pooling_dim * 2) self.post_emb.weight.data.normal_(0, math.sqrt(1.0)) self.rel_compress = nn.Linear(4096 + 256, 51, bias=True) self.rel_compress.weight = torch.nn.init.xavier_normal( self.rel_compress.weight, gain=1.0) self.node_transform = nn.Linear(4096, 256, bias=True) self.edge_transform = nn.Linear(4096, 256, bias=True) # self.rel_compress = CosineLinear(self.pooling_dim+256, self.num_rels) # self.rel_compress.weight = torch.nn.init.xavier_normal(self.rel_compress.weight, gain=1.0) if self.use_bias: self.freq_bias = FrequencyBias() if self.gnn: self.graph_network_node = GraphNetwork(4096) self.graph_network_edge = GraphNetwork() if self.training: self.graph_network_node.train() self.graph_network_edge.train() else: self.graph_network_node.eval() self.graph_network_edge.eval() self.edge_sim_network = nn.Linear(4096, 1, bias=True) self.metric_net = MetricLearning()
def __init__(self, classes, rel_classes, mode='sgdet', num_gpus=1, use_vision=True, require_overlap_det=True, embed_dim=200, hidden_dim=256, pooling_dim=4096, nl_obj=1, nl_edge=2, use_resnet=False, order='confidence', thresh=0.01, use_proposals=False, pass_in_obj_feats_to_decoder=True, pass_in_obj_feats_to_edge=True, rec_dropout=0.0, use_bias=True, use_tanh=True, limit_vision=True): """ :param classes: Object classes :param rel_classes: Relationship classes. None if were not using rel mode :param mode: (sgcls, predcls, or sgdet) :param num_gpus: how many GPUS 2 use :param use_vision: Whether to use vision in the final product :param require_overlap_det: Whether two objects must intersect :param embed_dim: Dimension for all embeddings :param hidden_dim: LSTM hidden size :param obj_dim: """ super(RelModelLinknet, self).__init__() self.classes = classes self.rel_classes = rel_classes self.num_gpus = num_gpus assert mode in MODES self.mode = mode self.pooling_size = 7 self.embed_dim = embed_dim self.hidden_dim = hidden_dim self.obj_dim = 2048 if use_resnet else 4096 self.ctx_dim = 1024 if use_resnet else 512 self.pooling_dim = pooling_dim self.use_bias = use_bias self.use_vision = use_vision self.use_tanh = use_tanh self.limit_vision = limit_vision self.require_overlap = require_overlap_det and self.mode == 'sgdet' self.detector = ObjectDetector( classes=classes, mode=('proposals' if use_proposals else 'refinerels') if mode == 'sgdet' else 'gtbox', use_resnet=use_resnet, thresh=thresh, max_per_img=64, ) self.context = LinearizedContext(self.classes, self.rel_classes, mode=self.mode, embed_dim=self.embed_dim, hidden_dim=self.hidden_dim, obj_dim=self.obj_dim, pooling_dim=self.pooling_dim, ctx_dim=self.ctx_dim) self.union_boxes = UnionBoxesAndFeats(pooling_size=self.pooling_size, stride=16, dim=1024 if use_resnet else 512) if use_resnet: self.roi_fmap = nn.Sequential( resnet_l4(relu_end=False), nn.AvgPool2d(self.pooling_size), Flattener(), ) else: roi_fmap = [ Flattener(), load_vgg(use_dropout=False, use_relu=False, use_linear=pooling_dim == 4096, pretrained=False).classifier, ] if pooling_dim != 4096: roi_fmap.append(nn.Linear(4096, pooling_dim)) self.roi_fmap = nn.Sequential(*roi_fmap) self.roi_fmap_obj = load_vgg(pretrained=False).classifier # Global Context Encoding self.GCE = GlobalContextEncoding(num_classes=self.num_classes, ctx_dim=self.ctx_dim) ################################### # K2 self.pos_embed = nn.Sequential(*[ nn.BatchNorm1d(4, momentum=BATCHNORM_MOMENTUM / 10.0), nn.Linear(4, 128), nn.ReLU(inplace=True), nn.Dropout(0.1), ]) # fc4 self.rel_compress = nn.Linear(self.pooling_dim + 128, self.num_rels, bias=True) self.rel_compress.weight = torch.nn.init.xavier_normal( self.rel_compress.weight, gain=1.0) if self.use_bias: self.freq_bias = FrequencyBias()
def __init__(self, classes, rel_classes, mode='sgdet', num_gpus=1, use_vision=True, require_overlap_det=True, embed_dim=200, hidden_dim=256, pooling_dim=2048, nl_obj=1, nl_edge=2, use_resnet=False, order='confidence', thresh=0.01, use_proposals=False, pass_in_obj_feats_to_decoder=True, pass_in_obj_feats_to_edge=True, rec_dropout=0.0, use_bias=True, use_tanh=True, limit_vision=True): """ :param classes: Object classes :param rel_classes: Relationship classes. None if were not using rel mode :param mode: (sgcls, predcls, or sgdet) :param num_gpus: how many GPUS 2 use :param use_vision: Whether to use vision in the final product :param require_overlap_det: Whether two objects must intersect :param embed_dim: Dimension for all embeddings :param hidden_dim: LSTM hidden size :param obj_dim: """ super(RelModel, self).__init__() self.classes = classes self.rel_classes = rel_classes self.num_gpus = num_gpus assert mode in MODES self.mode = mode self.pooling_size = 7 self.embed_dim = embed_dim self.hidden_dim = hidden_dim self.obj_dim = 2048 if use_resnet else 4096 self.pooling_dim = pooling_dim self.use_bias = use_bias self.use_vision = use_vision self.use_tanh = use_tanh self.limit_vision = limit_vision self.require_overlap = require_overlap_det and self.mode == 'sgdet' self.hook_for_grad = False self.gradients = [] self.detector = ObjectDetector( classes=classes, mode=('proposals' if use_proposals else 'refinerels') if mode == 'sgdet' else 'gtbox', use_resnet=use_resnet, thresh=thresh, max_per_img=64, ) self.ort_embedding = torch.autograd.Variable( get_ort_embeds(self.num_classes, 200).cuda()) embed_vecs = obj_edge_vectors(self.classes, wv_dim=self.embed_dim) self.obj_embed = nn.Embedding(self.num_classes, self.embed_dim) self.obj_embed.weight.data = embed_vecs.clone() # This probably doesn't help it much self.pos_embed = nn.Sequential(*[ nn.BatchNorm1d(4, momentum=BATCHNORM_MOMENTUM / 10.0), nn.Linear(4, 128), nn.ReLU(inplace=True), nn.Dropout(0.1), ]) self.context = LinearizedContext( self.classes, self.rel_classes, mode=self.mode, embed_dim=self.embed_dim, hidden_dim=self.hidden_dim, obj_dim=self.obj_dim, nl_obj=nl_obj, nl_edge=nl_edge, dropout_rate=rec_dropout, order=order, pass_in_obj_feats_to_decoder=pass_in_obj_feats_to_decoder, pass_in_obj_feats_to_edge=pass_in_obj_feats_to_edge) # Image Feats (You'll have to disable if you want to turn off the features from here) self.union_boxes = UnionBoxesAndFeats(pooling_size=self.pooling_size, stride=16, dim=1024 if use_resnet else 512) self.merge_obj_feats = nn.Sequential( nn.Linear(self.obj_dim + self.embed_dim + 128, self.hidden_dim), nn.ReLU()) # self.trans = nn.Sequential(nn.Linear(self.hidden_dim, self.hidden_dim//4), # LayerNorm(self.hidden_dim//4), nn.ReLU(), # nn.Linear(self.hidden_dim//4, self.hidden_dim)) self.get_phr_feats = nn.Linear(self.pooling_dim, self.hidden_dim) self.embeddings4lstm = nn.Embedding(self.num_classes, self.embed_dim) self.lstm = nn.LSTM(input_size=self.hidden_dim + self.embed_dim, hidden_size=self.hidden_dim, num_layers=1) self.obj_mps1 = Message_Passing4OBJ(self.hidden_dim) # self.obj_mps2 = Message_Passing4OBJ(self.hidden_dim) self.get_boxes_encode = Boxes_Encode(64) if use_resnet: self.roi_fmap = nn.Sequential( resnet_l4(relu_end=False), nn.AvgPool2d(self.pooling_size), Flattener(), ) else: roi_fmap = [ Flattener(), load_vgg(use_dropout=False, use_relu=False, use_linear=pooling_dim == 4096, pretrained=False).classifier, ] if pooling_dim != 4096: roi_fmap.append(nn.Linear(4096, pooling_dim)) self.roi_fmap = nn.Sequential(*roi_fmap) self.roi_fmap_obj = load_vgg(pretrained=False).classifier ################################### # self.obj_classify_head = nn.Linear(self.pooling_dim, self.num_classes) # self.post_emb_s = nn.Linear(self.pooling_dim, self.pooling_dim//2) # self.post_emb_s.weight = torch.nn.init.xavier_normal(self.post_emb_s.weight, gain=1.0) # self.post_emb_o = nn.Linear(self.pooling_dim, self.pooling_dim//2) # self.post_emb_o.weight = torch.nn.init.xavier_normal(self.post_emb_o.weight, gain=1.0) # self.merge_obj_high = nn.Linear(self.hidden_dim, self.pooling_dim//2) # self.merge_obj_high.weight = torch.nn.init.xavier_normal(self.merge_obj_high.weight, gain=1.0) # self.merge_obj_low = nn.Linear(self.pooling_dim + 5 + self.embed_dim, self.pooling_dim//2) # self.merge_obj_low.weight = torch.nn.init.xavier_normal(self.merge_obj_low.weight, gain=1.0) # self.rel_compress = nn.Linear(self.pooling_dim//2 + 64, self.num_rels, bias=True) # self.rel_compress.weight = torch.nn.init.xavier_normal(self.rel_compress.weight, gain=1.0) # self.freq_gate = nn.Linear(self.pooling_dim//2 + 64, self.num_rels, bias=True) # self.freq_gate.weight = torch.nn.init.xavier_normal(self.freq_gate.weight, gain=1.0) self.post_emb_s = nn.Linear(self.pooling_dim, self.pooling_dim) self.post_emb_s.weight = torch.nn.init.xavier_normal( self.post_emb_s.weight, gain=1.0) self.post_emb_o = nn.Linear(self.pooling_dim, self.pooling_dim) self.post_emb_o.weight = torch.nn.init.xavier_normal( self.post_emb_o.weight, gain=1.0) self.merge_obj_high = nn.Linear(self.hidden_dim, self.pooling_dim) self.merge_obj_high.weight = torch.nn.init.xavier_normal( self.merge_obj_high.weight, gain=1.0) self.merge_obj_low = nn.Linear(self.pooling_dim + 5 + self.embed_dim, self.pooling_dim) self.merge_obj_low.weight = torch.nn.init.xavier_normal( self.merge_obj_low.weight, gain=1.0) self.rel_compress = nn.Linear(self.pooling_dim + 64, self.num_rels, bias=True) self.rel_compress.weight = torch.nn.init.xavier_normal( self.rel_compress.weight, gain=1.0) self.freq_gate = nn.Linear(self.pooling_dim + 64, self.num_rels, bias=True) self.freq_gate.weight = torch.nn.init.xavier_normal( self.freq_gate.weight, gain=1.0) # self.ranking_module = nn.Sequential(nn.Linear(self.pooling_dim + 64, self.hidden_dim), nn.ReLU(), nn.Linear(self.hidden_dim, 1)) if self.use_bias: self.freq_bias = FrequencyBias()
def __init__(self, classes, rel_classes, mode='sgdet', num_gpus=1, use_vision=True, require_overlap_det=True, embed_dim=200, hidden_dim=256, pooling_dim=2048, nl_obj=1, nl_edge=2, use_resnet=False, order='confidence', thresh=0.01, use_proposals=False, pass_in_obj_feats_to_decoder=True, pass_in_obj_feats_to_edge=True, rec_dropout=0.0, use_bias=True, use_tanh=True, limit_vision=True): """ Args: classes: list, list of 151 object class names(including background) rel_classes: list, list of 51 predicate names( including background(norelationship)) mode: string, 'sgdet', 'predcls' or 'sgcls' num_gpus: integer, number of GPUs to use use_vision: boolean, whether to use vision in the final product require_overlap_det: boolean, whether two object must intersect embed_dim: integer, number of dimension for all embeddings hidden_dim: integer, hidden size of LSTM pooling_dim: integer, outputsize of vgg fc layer nl_obj: integer, number of object context layer, 2 in paper nl_edge: integer, number of edge context layer, 4 in paper use_resnet: integer, use resnet for backbone order: string, value must be in ('size', 'confidence', 'random', 'leftright'), order of RoIs thresh: float, threshold for scores of boxes if score of box smaller than thresh, then it will be abandoned use_proposals: boolean, whether to use proposals pass_in_obj_feats_to_decoder: boolean, whether to pass object features to decoder RNN pass_in_obj_feats_to_edge: boolean, whether to pass object features to edge context RNN rec_dropout: float, dropout rate in RNN use_bias: boolean, use_tanh: boolean, limit_vision: boolean, """ super(RelModel, self).__init__() self.classes = classes self.rel_classes = rel_classes self.num_gpus = num_gpus assert mode in MODES self.mode = mode self.pooling_size = 7 self.embed_dim = embed_dim self.hidden_dim = hidden_dim self.obj_dim = 2048 if use_resnet else 4096 self.pooling_dim = pooling_dim self.use_bias = use_bias self.use_vision = use_vision self.use_tanh = use_tanh self.limit_vision = limit_vision self.require_overlap = require_overlap_det and self.mode == 'sgdet' self.detector = ObjectDetector( classes=classes, mode=('proposals' if use_proposals else 'refinerels') if mode == 'sgdet' else 'gtbox', use_resnet=use_resnet, thresh=thresh, max_per_img=64, ) self.context = LinearizedContext( self.classes, self.rel_classes, mode=self.mode, embed_dim=self.embed_dim, hidden_dim=self.hidden_dim, obj_dim=self.obj_dim, nl_obj=nl_obj, nl_edge=nl_edge, dropout_rate=rec_dropout, order=order, pass_in_obj_feats_to_decoder=pass_in_obj_feats_to_decoder, pass_in_obj_feats_to_edge=pass_in_obj_feats_to_edge) # Image Feats (You'll have to disable if you want to turn off the features from here) self.union_boxes = UnionBoxesAndFeats(pooling_size=self.pooling_size, stride=16, dim=1024 if use_resnet else 512) if use_resnet: self.roi_fmap = nn.Sequential( resnet_l4(relu_end=False), nn.AvgPool2d(self.pooling_size), Flattener(), ) else: roi_fmap = [ Flattener(), load_vgg(use_dropout=False, use_relu=False, use_linear=pooling_dim == 4096, pretrained=False).classifier, ] if pooling_dim != 4096: roi_fmap.append(nn.Linear(4096, pooling_dim)) self.roi_fmap = nn.Sequential(*roi_fmap) self.roi_fmap_obj = load_vgg(pretrained=False).classifier ################################### self.post_lstm = nn.Linear(self.hidden_dim, self.pooling_dim * 2) # Initialize to sqrt(1/2n) so that the outputs all have mean 0 and variance 1. # (Half contribution comes from LSTM, half from embedding. # In practice the pre-lstm stuff tends to have stdev 0.1 so I multiplied this by 10. self.post_lstm.weight.data.normal_( 0, 10.0 * math.sqrt(1.0 / self.hidden_dim)) self.post_lstm.bias.data.zero_() if nl_edge == 0: self.post_emb = nn.Embedding(self.num_classes, self.pooling_dim * 2) self.post_emb.weight.data.normal_(0, math.sqrt(1.0)) self.rel_compress = nn.Linear(self.pooling_dim, self.num_rels, bias=True) self.rel_compress.weight = torch.nn.init.xavier_normal( self.rel_compress.weight, gain=1.0) if self.use_bias: self.freq_bias = FrequencyBias()
def __init__(self, classes, rel_classes, mode='sgdet', num_gpus=1, use_vision=True, require_overlap_det=True, embed_dim=200, hidden_dim=256, pooling_dim=2048, nl_obj=1, nl_edge=2, use_resnet=False, order='confidence', thresh=0.01, use_proposals=False, pass_in_obj_feats_to_decoder=True, model_path='', reachability=False, pass_in_obj_feats_to_edge=True, rec_dropout=0.0, use_bias=True, use_tanh=True, init_center=False, limit_vision=True): super(RelModel, self).__init__() self.classes = classes self.rel_classes = rel_classes self.num_gpus = num_gpus assert mode in MODES self.mode = mode self.init_center = init_center self.pooling_size = 7 self.model_path = model_path self.embed_dim = embed_dim self.hidden_dim = hidden_dim self.obj_dim = 2048 if use_resnet else 4096 self.pooling_dim = pooling_dim self.centroids = None self.use_bias = use_bias self.use_vision = use_vision self.use_tanh = use_tanh self.limit_vision = limit_vision self.require_overlap = require_overlap_det and self.mode == 'sgdet' self.global_embedding = EmbeddingImagenet(4096) self.detector = ObjectDetector( classes=classes, mode=('proposals' if use_proposals else 'refinerels') if mode == 'sgdet' else 'gtbox', use_resnet=use_resnet, thresh=thresh, max_per_img=64, ) self.context = LinearizedContext( self.classes, self.rel_classes, mode=self.mode, embed_dim=self.embed_dim, hidden_dim=self.hidden_dim, obj_dim=self.obj_dim, nl_obj=nl_obj, nl_edge=nl_edge, dropout_rate=rec_dropout, order=order, pass_in_obj_feats_to_decoder=pass_in_obj_feats_to_decoder, pass_in_obj_feats_to_edge=pass_in_obj_feats_to_edge) # Image Feats (You'll have to disable if you want to turn off the features from here) self.union_boxes = UnionBoxesAndFeats(pooling_size=self.pooling_size, stride=16, dim=1024 if use_resnet else 512) if use_resnet: self.roi_fmap = nn.Sequential( resnet_l4(relu_end=False), nn.AvgPool2d(self.pooling_size), Flattener(), ) else: roi_fmap = [ Flattener(), load_vgg(use_dropout=False, use_relu=False, use_linear=pooling_dim == 4096, pretrained=False).classifier, ] if pooling_dim != 4096: roi_fmap.append(nn.Linear(4096, pooling_dim)) self.roi_fmap = nn.Sequential(*roi_fmap) self.roi_fmap_obj = load_vgg(pretrained=False).classifier ################################### self.post_lstm = nn.Linear(self.hidden_dim, self.pooling_dim * 2) self.disc_center = DiscCentroidsLoss(self.num_rels, self.pooling_dim) self.meta_classify = MetaEmbedding_Classifier( feat_dim=self.pooling_dim, num_classes=self.num_rels) self.disc_center_g = DiscCentroidsLoss(self.num_classes, self.pooling_dim) self.meta_classify_g = MetaEmbedding_Classifier( feat_dim=self.pooling_dim, num_classes=self.num_classes) self.global_sub_additive = nn.Linear(4096, 1, bias=True) self.global_obj_additive = nn.Linear(4096, 1, bias=True) # Initialize to sqrt(1/2n) so that the outputs all have mean 0 and variance 1. # (Half contribution comes from LSTM, half from embedding. # In practice the pre-lstm stuff tends to have stdev 0.1 so I multiplied this by 10. self.post_lstm.weight.data.normal_( 0, 10.0 * math.sqrt(1.0 / self.hidden_dim)) self.post_lstm.bias.data.zero_() self.global_logist = nn.Linear(self.pooling_dim, self.num_classes, bias=True) # CosineLinear(4096,150)# self.global_logist.weight = torch.nn.init.xavier_normal( self.global_logist.weight, gain=1.0) self.post_emb = nn.Embedding(self.num_classes, self.pooling_dim * 2) self.post_emb.weight.data.normal_(0, math.sqrt(1.0)) self.rel_compress = nn.Linear(self.pooling_dim, self.num_rels, bias=True) self.rel_compress.weight = torch.nn.init.xavier_normal( self.rel_compress.weight, gain=1.0) if self.use_bias: self.freq_bias = FrequencyBias() self.class_num = torch.zeros(len(self.classes)) self.centroids = torch.zeros(len(self.classes), self.pooling_dim).cuda()
def __init__(self, classes, rel_classes, mode='sgdet', num_gpus=1, require_overlap_det=True, embed_dim=200, use_resnet=False, order='confidence', thresh=0.01, use_proposals=False): """ :param classes: Object classes :param rel_classes: Relationship classes. None if were not using rel mode :param mode: (sgcls, predcls, or sgdet) """ super(NODIS, self).__init__() self.classes = classes self.rel_classes = rel_classes self.num_gpus = num_gpus assert mode in MODES self.mode = mode self.pooling_size = 7 self.embed_dim = embed_dim self.obj_dim = 2048 if use_resnet else 4096 self.order = 'random' self.require_overlap = require_overlap_det and self.mode == 'sgdet' self.detector = ObjectDetector( classes=classes, mode=('proposals' if use_proposals else 'refinerels') if mode == 'sgdet' else 'gtbox', use_resnet=use_resnet, thresh=thresh, max_per_img=64, ) self.context = O_NODE(self.classes, self.rel_classes, mode=self.mode, embed_dim=self.embed_dim, obj_dim=self.obj_dim, order=order) # Image Feats (You'll have to disable if you want to turn off the features from here) self.union_boxes = UnionBoxesAndFeats(pooling_size=self.pooling_size, stride=16, dim=1024 if use_resnet else 512) if use_resnet: self.roi_fmap = nn.Sequential( resnet_l4(relu_end=False), nn.AvgPool2d(self.pooling_size), Flattener(), ) else: self.roi_fmap_obj = load_vgg(pretrained=False).classifier self.roi_avg_pool = nn.AvgPool2d(kernel_size=7, stride=0) ################################### embed_vecs = obj_edge_vectors(self.classes, wv_dim=self.embed_dim) self.obj_embed = nn.Embedding(self.num_classes, self.embed_dim) self.obj_embed.weight.data = embed_vecs.clone() self.obj_embed2 = nn.Embedding(self.num_classes, self.embed_dim) self.obj_embed2.weight.data = embed_vecs.clone() self.lstm_visual = nn.LSTM(input_size=1536, hidden_size=512) self.lstm_semantic = nn.LSTM(input_size=400, hidden_size=512) self.odeBlock = odeBlock(odeFunc1(bidirectional=True)) self.fc_predicate = nn.Sequential(nn.Linear(1024, 512), nn.ReLU(inplace=False), nn.Linear(512, 51), nn.ReLU(inplace=False))
def __init__(self, classes, rel_classes, mode='sgdet', num_gpus=1, use_vision=True, require_overlap_det=True, embed_dim=200, hidden_dim=256, obj_dim=2048, pooling_dim=4096, nl_obj=1, nl_edge=2, use_resnet=True, order='confidence', thresh=0.01, use_proposals=False, pass_in_obj_feats_to_decoder=True, pass_in_obj_feats_to_edge=True, rec_dropout=0.0, use_bias=True, use_tanh=True, limit_vision=True, spatial_dim=128, graph_constrain=True, mp_iter_num=1): """ Args: mp_iter_num: integer, number of message passing iteration """ super(FckModel, self).__init__() self.classes = classes self.rel_classes = rel_classes self.num_gpus = num_gpus assert mode in MODES self.mode = mode self.pooling_size = 7 self.embed_dim = embed_dim self.hidden_dim = hidden_dim self.obj_dim = obj_dim self.pooling_dim = 2048 if use_resnet else 4096 self.spatial_dim = spatial_dim self.use_bias = use_bias self.use_vision = use_vision self.use_tanh = use_tanh self.limit_vision = limit_vision self.require_overlap = require_overlap_det and self.mode == 'sgdet' self.graph_cons = graph_constrain self.mp_iter_num = mp_iter_num classes_word_vec = obj_edge_vectors(self.classes, wv_dim=embed_dim) self.classes_word_embedding = nn.Embedding(self.num_classes, embed_dim) self.classes_word_embedding.weight.data = classes_word_vec.clone() self.classes_word_embedding.weight.requires_grad = False # the last one is dirty bit self.rel_mem = nn.Embedding(self.num_rels, self.obj_dim + 1) self.rel_mem.weight.data[:, -1] = 0 if mode == 'sgdet': if use_proposals: obj_detector_mode = 'proposals' else: obj_detector_mode = 'refinerels' else: obj_detector_mode = 'gtbox' self.detector = ObjectDetector( classes=classes, mode=obj_detector_mode, use_resnet=use_resnet, thresh=thresh, max_per_img=64, ) self.union_boxes = UnionBoxesAndFeats(pooling_size=self.pooling_size, stride=16, dim=1024 if use_resnet else 512, use_feats=False) self.spatial_fc = nn.Sequential(*[ nn.Linear(4, spatial_dim), nn.BatchNorm1d(spatial_dim, momentum=BATCHNORM_MOMENTUM / 10.), nn.ReLU(inplace=True) ]) self.word_fc = nn.Sequential(*[ nn.Linear(2 * embed_dim, hidden_dim), nn.BatchNorm1d(hidden_dim, momentum=BATCHNORM_MOMENTUM / 10.), nn.ReLU(inplace=True) ]) # union box feats feats_dim = obj_dim + spatial_dim + hidden_dim self.relpn_fc = nn.Linear(feats_dim, 2) self.relcnn_fc1 = nn.Sequential( *[nn.Linear(feats_dim, feats_dim), nn.ReLU(inplace=True)]) self.box_mp_fc = nn.Sequential(*[ nn.Linear(obj_dim, obj_dim), ]) self.sub_rel_mp_fc = nn.Sequential(*[nn.Linear(feats_dim, obj_dim)]) self.obj_rel_mp_fc = nn.Sequential(*[ nn.Linear(feats_dim, obj_dim), ]) self.mp_atten_fc = nn.Sequential(*[ nn.Linear(feats_dim + obj_dim, obj_dim), nn.ReLU(inplace=True), nn.Linear(obj_dim, 1) ]) self.cls_fc = nn.Linear(obj_dim, self.num_classes) self.relcnn_fc2 = nn.Linear( feats_dim, self.num_rels if self.graph_cons else 2 * self.num_rels) if use_resnet: #deprecate self.roi_fmap = nn.Sequential( resnet_l4(relu_end=False), nn.AvgPool2d(self.pooling_size), Flattener(), ) else: roi_fmap = [ load_vgg( use_dropout=False, use_relu=False, use_linear=self.obj_dim == 4096, pretrained=False, ).classifier, nn.Linear(self.pooling_dim, self.obj_dim) ] self.roi_fmap = nn.Sequential(*roi_fmap)
def __init__(self, classes, rel_classes, mode='sgdet', num_gpus=1, use_vision=True, require_overlap_det=True, embed_dim=200, hidden_dim=256, pooling_dim=2048, nl_obj=1, nl_edge=2, use_resnet=False, order='confidence', thresh=0.01, use_proposals=False, pass_in_obj_feats_to_decoder=True, pass_in_obj_feats_to_edge=True, rec_dropout=0.0, use_bias=True, use_tanh=True, limit_vision=True): """ :param classes: Object classes :param rel_classes: Relationship classes. None if were not using rel mode :param mode: (sgcls, predcls, or sgdet) :param num_gpus: how many GPUS 2 use :param use_vision: Whether to use vision in the final product :param require_overlap_det: Whether two objects must intersect :param embed_dim: Dimension for all embeddings :param hidden_dim: LSTM hidden size :param obj_dim: """ super(RelModel, self).__init__() self.classes = classes self.rel_classes = rel_classes self.num_gpus = num_gpus assert mode in MODES self.mode = mode self.pooling_size = 7 self.embed_dim = embed_dim self.hidden_dim = hidden_dim self.obj_dim = 2048 if use_resnet else 4096 self.pooling_dim = pooling_dim self.use_bias = use_bias self.use_vision = use_vision self.use_tanh = use_tanh self.limit_vision=limit_vision self.require_overlap = require_overlap_det and self.mode == 'sgdet' # print('REL MODEL CONSTRUCTOR: 1') self.detector = ObjectDetector( classes=classes, mode=('proposals' if use_proposals else 'refinerels') if mode == 'sgdet' else 'gtbox', use_resnet=use_resnet, thresh=thresh, max_per_img=64, ) # print('REL MODEL CONSTRUCTOR: 2') self.context = LinearizedContext(self.classes, self.rel_classes, mode=self.mode, embed_dim=self.embed_dim, hidden_dim=self.hidden_dim, obj_dim=self.obj_dim, nl_obj=nl_obj, nl_edge=nl_edge, dropout_rate=rec_dropout, order=order, pass_in_obj_feats_to_decoder=pass_in_obj_feats_to_decoder, pass_in_obj_feats_to_edge=pass_in_obj_feats_to_edge) # Image Feats (You'll have to disable if you want to turn off the features from here) self.union_boxes = UnionBoxesAndFeats(pooling_size=self.pooling_size, stride=16, dim=1024 if use_resnet else 512) # print('REL MODEL CONSTRUCTOR: 3') if use_resnet: self.roi_fmap = nn.Sequential( resnet_l4(relu_end=False), nn.AvgPool2d(self.pooling_size), Flattener(), ) else: roi_fmap = [ Flattener(), load_vgg(use_dropout=False, use_relu=False, use_linear=pooling_dim == 4096, pretrained=False).classifier, ] if pooling_dim != 4096: roi_fmap.append(nn.Linear(4096, pooling_dim)) self.roi_fmap = nn.Sequential(*roi_fmap) self.roi_fmap_obj = load_vgg(pretrained=False).classifier # print('REL MODEL CONSTRUCTOR: 4') ################################### self.post_lstm = nn.Linear(self.hidden_dim, self.pooling_dim * 2) # Initialize to sqrt(1/2n) so that the outputs all have mean 0 and variance 1. # (Half contribution comes from LSTM, half from embedding. # In practice the pre-lstm stuff tends to have stdev 0.1 so I multiplied this by 10. self.post_lstm.weight.data.normal_(0, 10.0 * math.sqrt(1.0 / self.hidden_dim)) self.post_lstm.bias.data.zero_() # print('REL MODEL CONSTRUCTOR: 5') if nl_edge == 0: self.post_emb = nn.Embedding(self.num_classes, self.pooling_dim*2) self.post_emb.weight.data.normal_(0, math.sqrt(1.0)) self.rel_compress = nn.Linear(self.pooling_dim, self.num_rels, bias=True) self.rel_compress.weight = torch.nn.init.xavier_normal(self.rel_compress.weight, gain=1.0) if self.use_bias: self.freq_bias = FrequencyBias()
def __init__(self, classes, rel_classes, mode='sgdet', num_gpus=1, use_vision=True, require_overlap_det=False, embed_dim=200, hidden_dim=256, obj_dim=2048, pooling_dim=4096, nl_obj=1, nl_edge=2, use_resnet=True, order='confidence', thresh=0.01, use_proposals=False, pass_in_obj_feats_to_decoder=True, pass_in_obj_feats_to_edge=True, rec_dropout=0.0, use_bias=True, use_tanh=True, limit_vision=True, spatial_dim=128, mp_iter_num=1, trim_graph=True): """ Args: mp_iter_num: integer, number of message passing iteration trim_graph: boolean, trim graph in rel pn """ super(FckModel, self).__init__() self.classes = classes self.rel_classes = rel_classes self.num_gpus = num_gpus assert mode in MODES self.mode = mode self.pooling_size = 7 self.embed_dim = embed_dim self.hidden_dim = hidden_dim self.obj_dim = obj_dim self.pooling_dim = 2048 if use_resnet else 4096 self.spatial_dim = spatial_dim self.use_bias = use_bias self.use_vision = use_vision self.use_tanh = use_tanh self.limit_vision = limit_vision self.require_overlap = require_overlap_det and self.mode == 'sgdet' self.mp_iter_num = mp_iter_num self.trim_graph = trim_graph classes_word_vec = obj_edge_vectors(self.classes, wv_dim=embed_dim) self.classes_word_embedding = nn.Embedding(self.num_classes, embed_dim) self.classes_word_embedding.weight.data = classes_word_vec.clone() self.classes_word_embedding.weight.requires_grad = False #fg_matrix, bg_matrix = get_counts() #rel_obj_distribution = fg_matrix / (fg_matrix.sum(2)[:, :, None] + 1e-5) #rel_obj_distribution = torch.FloatTensor(rel_obj_distribution) #rel_obj_distribution = rel_obj_distribution.view(-1, self.num_rels) # #self.rel_obj_distribution = nn.Embedding(rel_obj_distribution.size(0), self.num_rels) ## (#obj_class * #obj_class, #rel_class) #self.rel_obj_distribution.weight.data = rel_obj_distribution if mode == 'sgdet': if use_proposals: obj_detector_mode = 'proposals' else: obj_detector_mode = 'refinerels' else: obj_detector_mode = 'gtbox' self.detector = ObjectDetector( classes=classes, mode=obj_detector_mode, use_resnet=use_resnet, thresh=thresh, max_per_img=64, ) self.union_boxes = UnionBoxesAndFeats(pooling_size=self.pooling_size, stride=16, dim=1024 if use_resnet else 512, use_feats=False) self.spatial_fc = nn.Sequential(*[ nn.Linear(4, spatial_dim), nn.BatchNorm1d(spatial_dim, momentum=BATCHNORM_MOMENTUM / 10.), nn.ReLU(inplace=True) ]) self.word_fc = nn.Sequential(*[ nn.Linear(2 * embed_dim, hidden_dim), nn.BatchNorm1d(hidden_dim, momentum=BATCHNORM_MOMENTUM / 10.), nn.ReLU(inplace=True) ]) # union box feats feats_dim = obj_dim + spatial_dim + hidden_dim self.relpn_fc = nn.Linear(feats_dim, 2) self.relcnn_fc1 = nn.Sequential( *[nn.Linear(feats_dim, feats_dim), nn.ReLU(inplace=True)]) # v2 model--------- self.box_mp_fc = nn.Sequential(*[ nn.Linear(obj_dim, obj_dim), ]) self.sub_rel_mp_fc = nn.Sequential(*[nn.Linear(feats_dim, obj_dim)]) self.obj_rel_mp_fc = nn.Sequential(*[ nn.Linear(feats_dim, obj_dim), ]) self.mp_atten_fc = nn.Sequential(*[ nn.Linear(feats_dim + obj_dim, obj_dim), nn.ReLU(inplace=True), nn.Linear(obj_dim, 1) ]) # v2 model---------- self.cls_fc = nn.Linear(obj_dim, self.num_classes) self.relcnn_fc2 = nn.Linear(feats_dim, self.num_rels) # v3 model ----------- self.mem_module = MemoryRNN(classes=classes, rel_classes=rel_classes, inputs_dim=feats_dim, hidden_dim=hidden_dim, recurrent_dropout_probability=.0) # v3 model ----------- if use_resnet: # deprecate self.roi_fmap = nn.Sequential( resnet_l4(relu_end=False), nn.AvgPool2d(self.pooling_size), Flattener(), ) else: roi_fmap = [ load_vgg( use_dropout=False, use_relu=False, use_linear=self.obj_dim == 4096, pretrained=False, ).classifier, nn.Linear(self.pooling_dim, self.obj_dim) ] self.roi_fmap = nn.Sequential(*roi_fmap)
def __init__(self, classes, rel_classes, mode='sgdet', num_gpus=1, use_vision=False, require_overlap_det=True, embed_dim=200, hidden_dim=4096, use_resnet=False, thresh=0.01, use_proposals=False, use_bias=True, limit_vision=True, depth_model=None, pretrained_depth=False, active_features=None, frozen_features=None, use_embed=False, **kwargs): """ :param classes: object classes :param rel_classes: relationship classes. None if were not using rel mode :param mode: (sgcls, predcls, or sgdet) :param num_gpus: how many GPUS 2 use :param use_vision: enable the contribution of union of bounding boxes :param require_overlap_det: whether two objects must intersect :param embed_dim: word2vec embeddings dimension :param hidden_dim: dimension of the fusion hidden layer :param use_resnet: use resnet as faster-rcnn's backbone :param thresh: faster-rcnn related threshold (Threshold for calling it a good box) :param use_proposals: whether to use region proposal candidates :param use_bias: enable frequency bias :param limit_vision: use truncated version of UoBB features :param depth_model: provided architecture for depth feature extraction :param pretrained_depth: whether the depth feature extractor should be initialized with ImageNet weights :param active_features: what set of features should be enabled (e.g. 'vdl' : visual, depth, and location features) :param frozen_features: what set of features should be frozen (e.g. 'd' : depth) :param use_embed: use word2vec embeddings """ RelModelBase.__init__(self, classes, rel_classes, mode, num_gpus, require_overlap_det, active_features, frozen_features) self.pooling_size = 7 self.embed_dim = embed_dim self.hidden_dim = hidden_dim self.obj_dim = 2048 if use_resnet else 4096 self.use_vision = use_vision self.use_bias = use_bias self.limit_vision = limit_vision # -- Store depth related parameters assert depth_model in DEPTH_MODELS self.depth_model = depth_model self.pretrained_depth = pretrained_depth self.depth_pooling_dim = DEPTH_DIMS[self.depth_model] self.use_embed = use_embed self.detector = nn.Module() features_size = 0 # -- Check whether ResNet is selected as faster-rcnn's backbone if use_resnet: raise ValueError( "The current model does not support ResNet as the Faster-RCNN's backbone." ) """ *** DIFFERENT COMPONENTS OF THE PROPOSED ARCHITECTURE *** This is the part where the different components of the proposed relation detection architecture are defined. In the case of RGB images, we have class probability distribution features, visual features, and the location ones. If we are considering depth images as well, we augment depth features too. """ # -- Visual features if self.has_visual: # -- Define faster R-CNN network and it's related feature extractors self.detector = ObjectDetector( classes=classes, mode=('proposals' if use_proposals else 'refinerels') if mode == 'sgdet' else 'gtbox', use_resnet=use_resnet, thresh=thresh, max_per_img=64, ) self.roi_fmap_obj = load_vgg(pretrained=False).classifier # -- Define union features if self.use_vision: # -- UoBB pooling module self.union_boxes = UnionBoxesAndFeats( pooling_size=self.pooling_size, stride=16, dim=1024 if use_resnet else 512) # -- UoBB feature extractor roi_fmap = [ Flattener(), load_vgg(use_dropout=False, use_relu=False, use_linear=self.hidden_dim == 4096, pretrained=False).classifier, ] if self.hidden_dim != 4096: roi_fmap.append(nn.Linear(4096, self.hidden_dim)) self.roi_fmap = nn.Sequential(*roi_fmap) # -- Define visual features hidden layer self.visual_hlayer = nn.Sequential(*[ xavier_init(nn.Linear(self.obj_dim * 2, self.FC_SIZE_VISUAL)), nn.ReLU(inplace=True), nn.Dropout(0.8) ]) self.visual_scale = ScaleLayer(1.0) features_size += self.FC_SIZE_VISUAL # -- Location features if self.has_loc: # -- Define location features hidden layer self.location_hlayer = nn.Sequential(*[ xavier_init(nn.Linear(self.LOC_INPUT_SIZE, self.FC_SIZE_LOC)), nn.ReLU(inplace=True), nn.Dropout(0.1) ]) self.location_scale = ScaleLayer(1.0) features_size += self.FC_SIZE_LOC # -- Class features if self.has_class: if self.use_embed: # -- Define class embeddings embed_vecs = obj_edge_vectors(self.classes, wv_dim=self.embed_dim) self.obj_embed = nn.Embedding(self.num_classes, self.embed_dim) self.obj_embed.weight.data = embed_vecs.clone() classme_input_dim = self.embed_dim if self.use_embed else self.num_classes # -- Define Class features hidden layer self.classme_hlayer = nn.Sequential(*[ xavier_init( nn.Linear(classme_input_dim * 2, self.FC_SIZE_CLASS)), nn.ReLU(inplace=True), nn.Dropout(0.1) ]) self.classme_scale = ScaleLayer(1.0) features_size += self.FC_SIZE_CLASS # -- Depth features if self.has_depth: # -- Initialize depth backbone self.depth_backbone = DepthCNN(depth_model=self.depth_model, pretrained=self.pretrained_depth) # -- Create a relation head which is used to carry on the feature extraction # from RoIs of depth features self.depth_rel_head = self.depth_backbone.get_classifier() # -- Define depth features hidden layer self.depth_rel_hlayer = nn.Sequential(*[ xavier_init( nn.Linear(self.depth_pooling_dim * 2, self.FC_SIZE_DEPTH)), nn.ReLU(inplace=True), nn.Dropout(0.6), ]) self.depth_scale = ScaleLayer(1.0) features_size += self.FC_SIZE_DEPTH # -- Initialize frequency bias if needed if self.use_bias: self.freq_bias = FrequencyBias() # -- *** Fusion layer *** -- # -- A hidden layer for concatenated features (fusion features) self.fusion_hlayer = nn.Sequential(*[ xavier_init(nn.Linear(features_size, self.hidden_dim)), nn.ReLU(inplace=True), nn.Dropout(0.1) ]) # -- Final FC layer which predicts the relations self.rel_out = xavier_init( nn.Linear(self.hidden_dim, self.num_rels, bias=True)) # -- Freeze the user specified features if self.frz_visual: self.freeze_module(self.detector) self.freeze_module(self.roi_fmap_obj) self.freeze_module(self.visual_hlayer) if self.use_vision: self.freeze_module(self.roi_fmap) self.freeze_module(self.union_boxes.conv) if self.frz_class: self.freeze_module(self.classme_hlayer) if self.frz_loc: self.freeze_module(self.location_hlayer) if self.frz_depth: self.freeze_module(self.depth_backbone) self.freeze_module(self.depth_rel_head) self.freeze_module(self.depth_rel_hlayer)
def __init__(self, classes, rel_classes, mode='sgdet', use_vision=True, embed_dim=200, hidden_dim=256, obj_dim=2048, pooling_dim=2048, pooling_size=7, dropout_rate=0.2, use_bias=True, use_tanh=True, limit_vision=True, sl_pretrain=False, num_iter=-1, use_resnet=False, reduce_input=False, debug_type=None, post_nms_thresh=0.5): super(DynamicFilterContext, self).__init__() self.classes = classes self.rel_classes = rel_classes assert mode in MODES self.mode = mode self.use_vision = use_vision self.use_bias = use_bias self.use_tanh = use_tanh self.use_highway = True self.limit_vision = limit_vision self.pooling_dim = pooling_dim self.pooling_size = pooling_size self.nms_thresh = post_nms_thresh self.obj_compress = myNNLinear(self.pooling_dim, self.num_classes, bias=True) # self.roi_fmap_obj = load_vgg(pretrained=False).classifier roi_fmap_obj = [myNNLinear(512*self.pooling_size*self.pooling_size, 4096, bias=True), nn.ReLU(inplace=True), nn.Dropout(p=0.5), myNNLinear(4096, 4096, bias=True), nn.ReLU(inplace=True), nn.Dropout(p=0.5)] self.roi_fmap_obj = nn.Sequential(*roi_fmap_obj) if self.use_bias: self.freq_bias = FrequencyBias() self.reduce_dim = 256 self.reduce_obj_fmaps = nn.Conv2d(512, self.reduce_dim, kernel_size=1) similar_fun = [myNNLinear(self.reduce_dim*2, self.reduce_dim), nn.ReLU(inplace=True), myNNLinear(self.reduce_dim, 1)] self.similar_fun = nn.Sequential(*similar_fun) # roi_fmap = [Flattener(), # load_vgg(use_dropout=False, use_relu=False, use_linear=self.pooling_dim == 4096, pretrained=False).classifier,] # if self.pooling_dim != 4096: # roi_fmap.append(nn.Linear(4096, self.pooling_dim)) # self.roi_fmap = nn.Sequential(*roi_fmap) roi_fmap = [Flattener(), nn.Linear(self.reduce_dim*2*self.pooling_size*self.pooling_size, 4096, bias=True), nn.ReLU(inplace=True), nn.Dropout(p=0.5), nn.Linear(4096, 4096, bias=True)] self.roi_fmap = nn.Sequential(*roi_fmap) self.hidden_dim = hidden_dim self.rel_compress = myNNLinear(self.hidden_dim*3, self.num_rels) self.post_obj = myNNLinear(self.pooling_dim, self.hidden_dim*2) self.mapping_x = myNNLinear(self.hidden_dim*2, self.hidden_dim*3) self.reduce_rel_input = myNNLinear(self.pooling_dim, self.hidden_dim*3)