def create_and_check_for_token_classification( self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels ): config.num_labels = self.num_labels model = LongformerForTokenClassification(config=config) model.to(torch_device) model.eval() result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels) self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.num_labels))
def create_and_check_longformer_for_token_classification( self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels ): config.num_labels = self.num_labels model = LongformerForTokenClassification(config=config) model.to(torch_device) model.eval() loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels) result = { "loss": loss, "logits": logits, } self.parent.assertListEqual(list(result["logits"].size()), [self.batch_size, self.seq_length, self.num_labels]) self.check_loss_output(result)
DEVICE = torch.device("cuda") print(DEVICE) data = trim_entity_spans(convert_goldparse('data/Resumes.json')) total = len(data) train_data, val_data = data[:180], data[180:] train_d = ResumeDataset(train_data, TOKENIZER, tag2idx, MAX_LEN) val_d = ResumeDataset(val_data, TOKENIZER, tag2idx, MAX_LEN) train_sampler = RandomSampler(train_d) train_dl = DataLoader(train_d, sampler=train_sampler, batch_size=8) val_dl = DataLoader(val_d, batch_size=4) #model = BertForTokenClassification.from_pretrained(MODEL_NAME, num_labels=len(tag2idx)) model = LongformerForTokenClassification.from_pretrained( MODEL_NAME, num_labels=len(tag2idx)) model.to(DEVICE) optimizer_grouped_parameters = get_hyperparameters(model, True) optimizer = Adam(optimizer_grouped_parameters, lr=3e-5) train_and_val_model(model, TOKENIZER, optimizer, EPOCHS, idx2tag, tag2idx, MAX_GRAD_NORM, DEVICE, train_dl, val_dl) torch.save( {"model_state_dict": model.state_dict()}, f'{output_path}/model-state.bin', )
torch.tensor(train_label)) val_dataset = TensorDataset(torch.tensor(val_ids), torch.tensor(val_mask), torch.tensor(val_label)) train_dataloader = DataLoader(train_dataset, batch_size=2, shuffle=True) val_dataloader = DataLoader(val_dataset, batch_size=2, shuffle=True) # Model Setup config = LongformerConfig.from_pretrained(longformer_pretrained) config.num_labels = 5 device = 'cuda:0' if torch.cuda.is_available() else 'cpu' print(device) model = LongformerForTokenClassification.from_pretrained( '../models/LF_targeted_sentiment/' ) # longformer_pretrained, config=config) model.cuda() # Model Train epochs = 20 optimizer = AdamW(model.parameters(), lr=1e-5, eps=1e-6) bias = [float(i) for i in '1,1,1'.split(',')] weight = (1 / torch.tensor(bias)).to(device) softmax = torch.nn.Softmax(dim=1) total_steps = len(train_dataloader) * epochs scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=1000, num_training_steps=total_steps)