def model_init(trial): """ Initializes models for hyperparameter searches. """ model_class = LxmertForQuestionAnswering model = LxmertForQuestionAnswering.from_pretrained(pretrained) return model
def __init__(self, device='cuda:0'): self.device = device # load models and model components frcnn_cfg = Config.from_pretrained("unc-nlp/frcnn-vg-finetuned") self.frcnn = GeneralizedRCNN.from_pretrained("unc-nlp/frcnn-vg-finetuned", config=frcnn_cfg).to(device) self.image_preprocess = Preprocess(frcnn_cfg) self.lxmert_tokenizer = LxmertTokenizer.from_pretrained("unc-nlp/lxmert-base-uncased") self.lxmert_gqa = LxmertForQuestionAnswering.from_pretrained("unc-nlp/lxmert-gqa-uncased").to(device)
def __init__(self): self.config = Config.from_pretrained("unc-nlp/frcnn-vg-finetuned") self.cnn = GeneralizedRCNN.from_pretrained( "unc-nlp/frcnn-vg-finetuned", config=self.config ) self.image_preprocess = Preprocess(self.config) self.tokenizer = LxmertTokenizer.from_pretrained("unc-nlp/lxmert-base-uncased") self.vqa = LxmertForQuestionAnswering.from_pretrained( "unc-nlp/lxmert-vqa-uncased" )
def create_and_check_lxmert_for_question_answering( self, config, input_ids, visual_feats, bounding_boxes, token_type_ids, input_mask, obj_labels, masked_lm_labels, matched_label, ans, output_attentions, ): model = LxmertForQuestionAnswering(config=config) model.to(torch_device) model.eval() result = model( input_ids, visual_feats, bounding_boxes, token_type_ids=token_type_ids, attention_mask=input_mask, labels=ans, output_attentions=output_attentions, return_dict=True, ) result = model(input_ids, visual_feats, bounding_boxes, labels=ans) result = model( input_ids, visual_feats, bounding_boxes, labels=ans, token_type_ids=token_type_ids, attention_mask=input_mask, output_attentions=output_attentions, ) result = model( input_ids, visual_feats, bounding_boxes, token_type_ids=token_type_ids, attention_mask=input_mask, labels=ans, output_attentions=not output_attentions, return_dict=True, ) self.parent.assertEqual(result.question_answering_score.shape, (self.batch_size, self.num_qa_labels))
def build_model_with_pretrain_weights(): logger.info("start loading %s" % (MINIVAL_QA_PATH)) val_img_ids, val_ques_ids, val_ques_inputs, val_labels, val_quesid2data = get_QA([ MINIVAL_QA_PATH]) val_generator = DataGenerator(val_img_ids, val_ques_ids, val_ques_inputs, val_labels, [VAL_IMGFEAT_PATH], BATCH_SIZE, False) logger.info("successfully build val generator") model = TFLxmertForQuestionAnswering() tf_weights = model.get_weights() lxmert_pytorch = LxmertForQuestionAnswering.from_pretrained( "unc-nlp/lxmert-vqa-uncased") pt_weights = [] for param, weight in zip(lxmert_pytorch.parameters(), tf_weights): if(param.data.shape == weight.shape): pt_weights.append(param.data) else: pt_weights.append(param.data.T) model.set_weights(pt_weights) # Run a validation loop at the end of each epoch. quesid2ans = {} for ques_ids, x_batch_val, y_batch_val in val_generator: val_logits = val_step(x_batch_val, y_batch_val, model) # batch_size * num_class # batch_size * 1 label = tf.argmax(val_logits, axis=1) for qid, l in zip(ques_ids, label): ans = LABEL2ANS[l] quesid2ans[qid.item()] = ans logger.info("\nVal accuracy: %0.2f\n" % (evaluate(quesid2ans, val_quesid2data) * 100.)) return
def resize_lxmert_num_qa_labels( self, config, input_ids, visual_feats, bounding_boxes, token_type_ids, input_mask, obj_labels, masked_lm_labels, matched_label, ans, output_attentions, ): start_labels = config.num_qa_labels num_large_labels = config.num_qa_labels * 2 num_small_labels = int(config.num_qa_labels * 2) less_labels_ans = ids_tensor([self.batch_size], num_small_labels) more_labels_ans = ids_tensor([self.batch_size], num_large_labels) model_pretrain = LxmertForPreTraining(config=config).to(torch_device) model_qa = LxmertForQuestionAnswering(config=config).to(torch_device) config.num_labels = num_small_labels end_labels = config.num_labels result_pretrain = model_pretrain( input_ids, visual_feats, bounding_boxes, token_type_ids=token_type_ids, attention_mask=input_mask, ans=ans, ) result_qa = model_qa( input_ids, visual_feats, bounding_boxes, labels=ans, token_type_ids=token_type_ids, attention_mask=input_mask, ) model_pretrain.resize_num_qa_labels(num_small_labels) model_qa.resize_num_qa_labels(num_small_labels) result_pretrain_less = model_pretrain( input_ids, visual_feats, bounding_boxes, token_type_ids=token_type_ids, attention_mask=input_mask, ans=less_labels_ans, ) result_qa_less = model_qa( input_ids, visual_feats, bounding_boxes, labels=less_labels_ans, token_type_ids=token_type_ids, attention_mask=input_mask, ) model_pretrain.resize_num_qa_labels(num_large_labels) model_qa.resize_num_qa_labels(num_large_labels) result_pretrain_more = model_pretrain( input_ids, visual_feats, bounding_boxes, token_type_ids=token_type_ids, attention_mask=input_mask, ans=more_labels_ans, ) result_qa_more = model_qa( input_ids, visual_feats, bounding_boxes, labels=more_labels_ans, token_type_ids=token_type_ids, attention_mask=input_mask, ) model_qa_labels = model_qa.num_qa_labels self.parent.assertNotEqual(start_labels, end_labels) self.parent.assertNotEqual(model_qa_labels, start_labels) self.parent.assertEqual(result_qa.question_answering_score.shape, (self.batch_size, start_labels)) self.parent.assertEqual(result_pretrain.question_answering_score.shape, (self.batch_size, start_labels)) self.parent.assertEqual(result_qa_less.question_answering_score.shape, (self.batch_size, num_small_labels)) self.parent.assertEqual( result_pretrain_less.question_answering_score.shape, (self.batch_size, num_small_labels)) self.parent.assertEqual(result_qa_more.question_answering_score.shape, (self.batch_size, num_large_labels)) self.parent.assertEqual( result_pretrain_more.question_answering_score.shape, (self.batch_size, num_large_labels))
dtype=dtype) item[key] = item[key].reshape(shape) item[key].setflags(write=False) # Normalize the boxes (to 0 ~ 1) item["normalized_boxes"] = item["boxes"].copy() img_h, img_w = item["img_h"], item["img_w"] item["normalized_boxes"][:, (0, 2)] /= img_w item["normalized_boxes"][:, (1, 3)] /= img_h item["normalized_boxes"].setflags(write=False) imgid2img[item["img_id"]] = item lxmert_tokenizer = LxmertTokenizer.from_pretrained( "unc-nlp/lxmert-base-uncased") lxmert_gqa = LxmertForQuestionAnswering.from_pretrained( "unc-nlp/lxmert-gqa-uncased").to("cuda") GQA_URL = "https://raw.githubusercontent.com/airsplay/lxmert/master/data/gqa/trainval_label2ans.json" gqa_answers = utils.get_data(GQA_URL) accuracy = 0 for question_id, question_obj in tqdm(dataset.items()): inputs = lxmert_tokenizer(question_obj["question"], return_token_type_ids=True, return_attention_mask=True, add_special_tokens=True, return_tensors="pt").to("cuda") output_gqa = lxmert_gqa( input_ids=inputs.input_ids, attention_mask=inputs.attention_mask,