def create_and_check_for_token_classification(
self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
):
config.num_labels = self.num_labels
model = LongformerForTokenClassification(config=config)
model.to(torch_device)
model.eval()
result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels)
self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.num_labels))
def create_and_check_longformer_for_token_classification(
self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
):
config.num_labels = self.num_labels
model = LongformerForTokenClassification(config=config)
model.to(torch_device)
model.eval()
loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels)
result = {
"loss": loss,
"logits": logits,
}
self.parent.assertListEqual(list(result["logits"].size()), [self.batch_size, self.seq_length, self.num_labels])
self.check_loss_output(result)
DEVICE = torch.device("cuda")
print(DEVICE)
data = trim_entity_spans(convert_goldparse('data/Resumes.json'))
total = len(data)
train_data, val_data = data[:180], data[180:]
train_d = ResumeDataset(train_data, TOKENIZER, tag2idx, MAX_LEN)
val_d = ResumeDataset(val_data, TOKENIZER, tag2idx, MAX_LEN)
train_sampler = RandomSampler(train_d)
train_dl = DataLoader(train_d, sampler=train_sampler, batch_size=8)
val_dl = DataLoader(val_d, batch_size=4)
#model = BertForTokenClassification.from_pretrained(MODEL_NAME, num_labels=len(tag2idx))
model = LongformerForTokenClassification.from_pretrained(
MODEL_NAME, num_labels=len(tag2idx))
model.to(DEVICE)
optimizer_grouped_parameters = get_hyperparameters(model, True)
optimizer = Adam(optimizer_grouped_parameters, lr=3e-5)
train_and_val_model(model, TOKENIZER, optimizer, EPOCHS, idx2tag, tag2idx,
MAX_GRAD_NORM, DEVICE, train_dl, val_dl)
torch.save(
{"model_state_dict": model.state_dict()},
f'{output_path}/model-state.bin',
)
torch.tensor(train_label))
val_dataset = TensorDataset(torch.tensor(val_ids), torch.tensor(val_mask),
torch.tensor(val_label))
train_dataloader = DataLoader(train_dataset, batch_size=2, shuffle=True)
val_dataloader = DataLoader(val_dataset, batch_size=2, shuffle=True)
# Model Setup
config = LongformerConfig.from_pretrained(longformer_pretrained)
config.num_labels = 5
device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
print(device)
model = LongformerForTokenClassification.from_pretrained(
'../models/LF_targeted_sentiment/'
) # longformer_pretrained, config=config)
model.cuda()
# Model Train
epochs = 20
optimizer = AdamW(model.parameters(), lr=1e-5, eps=1e-6)
bias = [float(i) for i in '1,1,1'.split(',')]
weight = (1 / torch.tensor(bias)).to(device)
softmax = torch.nn.Softmax(dim=1)
total_steps = len(train_dataloader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps=1000,
num_training_steps=total_steps)