def load_untrained_model(args):
if args.dataset in ['simple', 'synth', 'comp']:
tokenizer = SimBertTokenizer()
if args.dataset == 'simple':
config = SimBertConfig()
elif args.dataset == 'comp':
config = CompBertConfig()
else:
config = SynBertConfig()
model = RobertaForQuestionAnswering(config=config, )
elif args.dataset in ['hpqa', 'squad']:
config = AutoConfig.from_pretrained(
args.config_name if args.config_name else args.model_name_or_path,
cache_dir=args.cache_dir if args.cache_dir else None,
)
tokenizer = AutoTokenizer.from_pretrained(
args.tokenizer_name
if args.tokenizer_name else args.model_name_or_path,
do_lower_case=args.do_lower_case,
cache_dir=args.cache_dir if args.cache_dir else None,
)
model = AutoModelForQuestionAnswering.from_pretrained(
args.model_name_or_path,
from_tf=bool(".ckpt" in args.model_name_or_path),
config=config,
cache_dir=args.cache_dir if args.cache_dir else None,
)
else:
raise RuntimeError('Dataset not supported')
return config, tokenizer, model
def create_model(self, transformer="longformer"):
if transformer == "distilbert":
from transformers import DistilBertForQuestionAnswering
self.model = DistilBertForQuestionAnswering.from_pretrained(
"distilbert-base-uncased")
elif transformer == "bert":
from transformers import BertForQuestionAnswering
self.model = BertForQuestionAnswering.from_pretrained(
"bert-base-uncased")
elif transformer == "roberta":
from transformers import RobertaForQuestionAnswering
self.model = RobertaForQuestionAnswering.from_pretrained(
"roberta-base")
elif transformer == "roberta_squad":
from transformers import RobertaForQuestionAnswering
self.model = RobertaForQuestionAnswering.from_pretrained(
"deepset/roberta-base-squad2")
elif transformer == "longformer":
from transformers import LongformerForQuestionAnswering
self.model = LongformerForQuestionAnswering.from_pretrained(
"allenai/longformer-base-4096")
elif transformer == "bart":
from transformers import BartForQuestionAnswering
self.model = BartForQuestionAnswering.from_pretrained(
"facebook/bart-base")
elif transformer == "electra":
from transformers import ElectraForQuestionAnswering
self.model = ElectraForQuestionAnswering.from_pretrained(
"google/electra-small-discriminator")
else:
print(
"The model you chose is not available in this version. You can try to manually change the code or manually overwrite the variable self.model"
)
print(
"The available choices are 'distilbert' , 'bert' , 'roberta' , 'longformer' , 'bart' , 'electra' "
)
def create_and_check_for_question_answering(
self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
):
model = RobertaForQuestionAnswering(config=config)
model.to(torch_device)
model.eval()
result = model(
input_ids,
attention_mask=input_mask,
token_type_ids=token_type_ids,
start_positions=sequence_labels,
end_positions=sequence_labels,
)
self.parent.assertEqual(result.start_logits.shape, (self.batch_size, self.seq_length))
self.parent.assertEqual(result.end_logits.shape, (self.batch_size, self.seq_length))
def configure_tokenizer_model_roberta(args, logger, is_preprocess=False):
logger.info("***** Loading tokenizer *****")
tokenizer = RobertaTokenizer.from_pretrained(args.config_name,
do_lower_case=args.do_lower_case,
cache_dir=args.init_dir,)
logger.info("***** Loading configuration *****")
config = RobertaConfig.from_pretrained(args.config_name, cache_dir=args.init_dir)
logger.info("Loading pretrained model from {}".format(args.init_dir))
if is_preprocess:
model = AutoModel.from_pretrained(args.model_name_or_path,
config=config,
cache_dir=args.init_dir)
else:
model = RobertaForQuestionAnswering.from_pretrained(args.init_dir, config=config,
cache_dir=args.init_dir)
return tokenizer, model
def create_and_check_roberta_for_question_answering(
self, config, input_ids, token_type_ids, input_mask,
sequence_labels, token_labels, choice_labels):
model = RobertaForQuestionAnswering(config=config)
model.to(torch_device)
model.eval()
result = model(
input_ids,
attention_mask=input_mask,
token_type_ids=token_type_ids,
start_positions=sequence_labels,
end_positions=sequence_labels,
)
self.parent.assertListEqual(list(result["start_logits"].size()),
[self.batch_size, self.seq_length])
self.parent.assertListEqual(list(result["end_logits"].size()),
[self.batch_size, self.seq_length])
self.check_loss_output(result)
def __init__(self, hparams):
super().__init__(hparams)
self.net = RobertaForQuestionAnswering.from_pretrained(
self.hparams.model)
