def build_image_attention_module(image_att_model_par, image_dim, ques_dim):
modal_combine_module = build_modal_combine_module(
image_att_model_par['modal_combine']['method'],
image_att_model_par['modal_combine']['par'],
image_feat_dim=image_dim, ques_emb_dim=ques_dim)
transform_module = build_post_combine_transform(
image_att_model_par['transform']['method'],
image_att_model_par['transform']['par'],
in_dim=modal_combine_module.out_dim)
normalization = image_att_model_par['normalization']
return top_down_attention(
modal_combine_module, normalization, transform_module)
def build_image_attention_module(image_att_model_par, image_dim, ques_dim):
modal_combine_module = build_modal_combine_module(
image_att_model_par['modal_combine']['method'],
image_att_model_par['modal_combine']['par'],
image_feat_dim=image_dim, ques_emb_dim=ques_dim)
transform_module = build_post_combine_transform(
image_att_model_par['transform']['method'],
image_att_model_par['transform']['par'],
in_dim=modal_combine_module.out_dim)
normalization = image_att_model_par['normalization']
return top_down_attention(
modal_combine_module, normalization, transform_module)
def build_image_attention_module(image_att_model_par, image_dim, ques_dim):
modal_combine_module = build_modal_combine_module(
image_att_model_par["modal_combine"]["method"],
image_att_model_par["modal_combine"]["par"],
image_feat_dim=image_dim,
ques_emb_dim=ques_dim,
)
transform_module = build_post_combine_transform(
image_att_model_par["transform"]["method"],
image_att_model_par["transform"]["par"],
in_dim=modal_combine_module.out_dim,
)
normalization = image_att_model_par["normalization"]
return top_down_attention(modal_combine_module, normalization,
transform_module)
def prepare_model(num_vocab_txt, num_choices, **model_config):
image_feat_dim = model_config['image_feat_dim']
# generate the list of question embedding models
ques_embeding_models_list = model_config['question_embedding']
question_embeding_models = nn.ModuleList()
final_question_embeding_dim = 0
for ques_embeding_model in ques_embeding_models_list:
ques_model_key = ques_embeding_model['method']
ques_model_par = ques_embeding_model['par']
tmp_model = build_question_encoding_module(ques_model_key,
ques_model_par,
num_vocab=num_vocab_txt)
question_embeding_models.append(tmp_model)
final_question_embeding_dim += tmp_model.text_out_dim
image_feature_encode_list = nn.ModuleList()
for image_feat_model_par in model_config['image_feature_encoding']:
image_feat_model = build_image_feature_encoding(
image_feat_model_par['method'], image_feat_model_par['par'],
image_feat_dim)
image_feature_encode_list.append(image_feat_model)
image_feat_dim = image_feat_model.out_dim
# generate the list of image attention models
image_emdedding_models_list = nn.ModuleList()
num_image_feat = model_config['num_image_feat']
final_image_embedding_dim = 0
for i_image in range(num_image_feat):
image_emdedding_models = nn.ModuleList()
image_att_model_list = model_config['image_embedding_models']
for image_att_model in image_att_model_list:
image_att_model_par = image_att_model
tmp_img_att_model = build_image_attention_module(
image_att_model_par,
image_dim=image_feat_dim,
ques_dim=final_question_embeding_dim)
tmp_img_model = image_embedding(tmp_img_att_model)
final_image_embedding_dim += tmp_img_model.out_dim
image_emdedding_models.append(tmp_img_model)
image_emdedding_models_list.append(image_emdedding_models)
final_image_embedding_dim *= image_feat_dim
inter_model = None
# parse multi-modal combination after image-embedding & question-embedding
multi_modal_combine = build_modal_combine_module(
model_config['modal_combine']['method'],
model_config['modal_combine']['par'], final_image_embedding_dim,
final_question_embeding_dim)
joint_embedding_dim = multi_modal_combine.out_dim
# generate the classifier
classifier = build_classifier(model_config['classifier']['method'],
model_config['classifier']['par'],
in_dim=joint_embedding_dim,
out_dim=num_choices)
my_model = vqa_multi_modal_model(image_emdedding_models_list,
question_embeding_models,
multi_modal_combine, classifier,
image_feature_encode_list, inter_model)
if use_cuda:
my_model = my_model.cuda()
if torch.cuda.device_count() > 1:
my_model = nn.DataParallel(my_model)
return my_model