def __init__(self, learning_rate: float, roberta_type: str = 'roberta-base'):
super().__init__()
config = RobertaConfig.from_pretrained(roberta_type)
config.num_labels = 2
self.num_labels = config.num_labels
self.config = config
self.lr = learning_rate
self.model = RobertaForMultipleChoice(config).from_pretrained(roberta_type, num_labels=self.num_labels)
def create_and_check_roberta_for_multiple_choice(
self, config, input_ids, token_type_ids, input_mask,
sequence_labels, token_labels, choice_labels):
config.num_choices = self.num_choices
model = RobertaForMultipleChoice(config=config)
model.to(torch_device)
model.eval()
multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(
-1, self.num_choices, -1).contiguous()
multiple_choice_token_type_ids = token_type_ids.unsqueeze(
1).expand(-1, self.num_choices, -1).contiguous()
multiple_choice_input_mask = input_mask.unsqueeze(1).expand(
-1, self.num_choices, -1).contiguous()
loss, logits = model(
multiple_choice_inputs_ids,
attention_mask=multiple_choice_input_mask,
token_type_ids=multiple_choice_token_type_ids,
labels=choice_labels,
)
result = {
"loss": loss,
"logits": logits,
}
self.parent.assertListEqual(list(result["logits"].size()),
[self.batch_size, self.num_choices])
self.check_loss_output(result)
def main(argv):
parser = argparse.ArgumentParser(description='')
required = parser.add_argument_group('required arguments')
required.add_argument('-r', '--retrieval', choices=['IR', 'NSP', 'NN'] , help='retrieval solver for the contexts. Options: IR, NSP or NN', required=True)
parser.add_argument('-t', '--dataset', default='ndq', choices=['ndq', 'dq'], help='dataset to train the model with. Options: ndq or dq. Default: ndq')
parser.add_argument('-d', '--device', default='gpu', choices=['gpu', 'cpu'], help='device to train the model with. Options: cpu or gpu. Default: gpu')
parser.add_argument('-p', '--pretrainings', default="checkpoints/pretrainings_e4.pth", help='path to the pretrainings model. If empty, the model will be the RobertForMultipleChoice with roberta-large weights. Default: checkpoints/pretrainings_e4.pth')
parser.add_argument('-b', '--batchsize', default= 1, type=int, help='size of the batches. Default: 1')
parser.add_argument('-x', '--maxlen', default= 180, type=int, help='max sequence length. Default: 180')
parser.add_argument('-l', '--lr', default= 1e-5, type=float, help='learning rate. Default: 1e-5')
parser.add_argument('-e', '--epochs', default= 4, type=int, help='number of epochs. Default: 4')
parser.add_argument('-s', '--save', default=False, help='save model at the end of the training', action='store_true')
args = parser.parse_args()
print(args)
if args.pretrainings == "":
model = RobertaForMultipleChoice.from_pretrained("roberta-large")
else:
model = torch.load(args.pretrainings)
tokenizer = RobertaTokenizer.from_pretrained('roberta-large')
if args.device=="gpu":
device = torch.device("cuda")
model.cuda()
if args.device=="cpu":
device = torch.device("cpu")
model.cpu()
model.zero_grad()
batch_size = args.batchsize
max_len = args.maxlen
lr = args.lr
epochs = args.epochs
retrieval_solver = args.retrieval
save_model = args.save
dataset_name = args.dataset
raw_data_train = get_data_ndq(dataset_name, "train", retrieval_solver, tokenizer, max_len)
raw_data_val = get_data_ndq(dataset_name, "val", retrieval_solver, tokenizer, max_len)
train_dataloader = process_data_ndq(raw_data_train, batch_size, "train")
val_dataloader = process_data_ndq(raw_data_val, batch_size, "val")
optimizer = AdamW(model.parameters(), lr = lr, eps = 1e-8)
total_steps = len(train_dataloader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps = 0, num_training_steps = total_steps)
training_ndq(model, train_dataloader, val_dataloader, optimizer, scheduler, epochs, retrieval_solver, device, save_model, dataset_name)
def __init__(self, model_name_or_path): super(Model, self).__init__() self.config = RobertaConfig.from_pretrained( model_name_or_path, num_labels=num_labels, ) self.tokenizer = RobertaTokenizer.from_pretrained( model_name_or_path, do_lower_case=False, ) self.model = RobertaForMultipleChoice.from_pretrained( model_name_or_path, from_tf=False, config=self.config, ) self.model = self.model.eval().cuda() self.m = nn.Softmax(1)
def create_and_check_for_multiple_choice(
self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
):
config.num_choices = self.num_choices
model = RobertaForMultipleChoice(config=config)
model.to(torch_device)
model.eval()
multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
multiple_choice_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
multiple_choice_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
result = model(
multiple_choice_inputs_ids,
attention_mask=multiple_choice_input_mask,
token_type_ids=multiple_choice_token_type_ids,
labels=choice_labels,
)
self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_choices))
def __init__(self, config):
super().__init__(config)
self.config = config
self.model = RobertaForMultipleChoice.from_pretrained("roberta-base")
class RobertaPIQA(pl.LightningModule):
def __init__(self, learning_rate: float, roberta_type: str = 'roberta-base'):
super().__init__()
config = RobertaConfig.from_pretrained(roberta_type)
config.num_labels = 2
self.num_labels = config.num_labels
self.config = config
self.lr = learning_rate
self.model = RobertaForMultipleChoice(config).from_pretrained(roberta_type, num_labels=self.num_labels)
# self.model.init_weights()
def forward(self, *args, **kwargs) -> MultipleChoiceModelOutput:
return self.model.forward(*args, **kwargs)
def training_step(self, batch, batch_idx):
# unpack batch
input = batch['input_ids']
mask = batch['attention_mask']
token_type = batch['token_type_ids']
label = batch['label']
# forward + loss
output = self.model(
input_ids=input,
attention_mask=mask,
token_type_ids=token_type,
labels=label)
loss = output.loss
out = torch.argmax(output.logits, dim=1)
correct = sum(out == label).item()
acc = correct / len(label)
# make so that the loss is logged
self.log('train_loss', loss)
self.log('train_accuracy', acc, on_epoch=True)
return loss
def validation_step(self, batch, batch_idx):
# unpack batch
input = batch['input_ids']
mask = batch['attention_mask']
token_type = batch['token_type_ids']
label = batch['label']
# forward + loss
output = self.model(
input_ids=input,
attention_mask=mask,
token_type_ids=token_type,
labels=label)
loss = output.loss
out = torch.argmax(output.logits, dim=1)
correct = sum(out == label).item()
acc = correct / len(label)
# make so that the loss is logged
self.log('val_loss', loss)
self.log('val_accuracy', acc, on_epoch=True, prog_bar=True)
return {'loss': loss, 'logits': output.logits, 'output': out, 'correct': correct}
def test_step(self, batch, batch_idx):
# unpack batch
input = batch['input_ids']
mask = batch['attention_mask']
token_type = batch['token_type_ids']
# forward + loss
output = self.model(
input_ids=input,
attention_mask=mask,
token_type_ids=token_type)
out = torch.argmax(output.logits, dim=1)
self.log(f'test_loss', 0.0)
return {'loss': 0.0, 'logits': output.logits, 'output': out}
def configure_optimizers(self):
return torch.optim.Adam(
self.parameters(),
lr=self.lr,
)
default=
"/net/nfs.websail/yyv959/winogrande/outputs/roberta-large/train-l-mc-fake-medium-sym-200000-unigram-8/",
type=str,
help=
"The input data dir. Should contain the .tsv files (or other data files) for the task."
)
parser.add_argument("--no_cuda",
action='store_true',
help="Avoid using CUDA when available")
args = parser.parse_args()
dir = args.dir
mc_model_path = args.mc_model_path
mc_model = RobertaForMultipleChoice.from_pretrained(mc_model_path)
mc_tokenizer = RobertaTokenizer.from_pretrained(mc_model_path)
mc_model.eval()
device = torch.device(
"cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
sent_encoder = SentenceTransformer('roberta-base-nli-stsb-mean-tokens',
device=device)
mc_model.to(device)
tagger = spacy.load("en_core_web_lg")
word_vector = gensim.models.KeyedVectors.load_word2vec_format(
'/net/nfs.websail/yyv959/counter-fitted-vectors.txt', binary=False)
def main(argv):
parser = argparse.ArgumentParser(description='')
parser.add_argument(
'-d',
'--device',
default='gpu',
choices=['gpu', 'cpu'],
help='device to train the model with. Options: cpu or gpu. Default: gpu'
)
parser.add_argument(
'-p',
'--pretrainings',
default='../checkpoints/RACE_e1.pth',
help=
'path to the pretrainings model. Default: ../checkpoints/RACE_e1.pth')
parser.add_argument('-b',
'--batchsize',
default=1,
type=int,
help='size of the batches. Default: 1')
parser.add_argument('-x',
'--maxlen',
default=256,
type=int,
help='max sequence length. Default: 256')
parser.add_argument('-l',
'--lr',
default=1e-5,
type=float,
help='learning rate. Default: 1e-5')
parser.add_argument('-e',
'--epochs',
default=4,
type=int,
help='number of epochs. Default: 4')
parser.add_argument('-s',
'--save',
default=False,
help='save model at the end of the training',
action='store_true')
args = parser.parse_args()
print(args)
if args.pretrainings == "":
model = RobertaForMultipleChoice.from_pretrained("roberta-large")
else:
model = torch.load(args.pretrainings)
tokenizer = RobertaTokenizer.from_pretrained('roberta-large')
if args.device == "gpu":
device = torch.device("cuda")
model.cuda()
if args.device == "cpu":
device = torch.device("cpu")
model.cpu()
model.zero_grad()
batch_size = args.batchsize
max_len = args.maxlen
dataset_name = "pretrainings"
lr = args.lr
epochs = args.epochs
save_model = args.save
raw_data_train = get_data_pretrainings(dataset_name, "train", tokenizer,
max_len)
raw_data_val = get_data_pretrainings(dataset_name, "val", tokenizer,
max_len)
train_dataloader = process_data_ndq(raw_data_train, batch_size, "train")
val_dataloader = process_data_ndq(raw_data_val, batch_size, "val")
optimizer = AdamW(model.parameters(), lr=lr, eps=1e-8)
total_steps = len(train_dataloader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps=0,
num_training_steps=total_steps)
training_ndq(model, train_dataloader, val_dataloader, optimizer, scheduler,
epochs, device, save_model, dataset_name)
