def main(): """Fine-tune bert""" train_data = reldata.RelData(os.path.join(base, cfg.get('data', 'xmi_dir')), partition='train', n_files=cfg.get('data', 'n_files')) tr_texts, tr_labels = train_data.event_time_relations() train_loader = make_data_loader(tr_texts, tr_labels, RandomSampler) val_data = reldata.RelData(os.path.join(base, cfg.get('data', 'xmi_dir')), partition='dev', n_files=cfg.get('data', 'n_files')) val_texts, val_labels = val_data.event_time_relations() val_loader = make_data_loader(val_texts, val_labels, SequentialSampler) print('loaded %d training and %d validation samples' % \ (len(tr_texts), len(val_texts))) model = TransformerClassifier() label_counts = torch.bincount(torch.IntTensor(tr_labels)) weights = len(tr_labels) / (2.0 * label_counts) train(model, train_loader, val_loader, weights) evaluate(model, val_loader, weights, suppress_output=False)
def main(): """Fine-tune bert""" model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=3) if torch.cuda.is_available(): device = torch.device('cuda') model.cuda() else: device = torch.device('cpu') model.cpu() train_data = reldata.RelData(os.path.join(base, cfg.get('data', 'xmi_dir')), partition='train') texts, labels = train_data.event_time_relations() train_loader = make_data_loader(texts, labels, RandomSampler) train(model, train_loader, device) dev_data = reldata.RelData(os.path.join(base, cfg.get('data', 'xmi_dir')), partition='dev') texts, labels = dev_data.event_time_relations() dev_loader = make_data_loader(texts, labels, sampler=SequentialSampler) evaluate(model, dev_loader, device)
def main(): """Fine-tune bert""" train_data = reldata.RelData(os.path.join(base, cfg.get('data', 'xmi_dir')), partition='train', n_files=cfg.get('data', 'n_files')) tr_texts, tr_labels = train_data.event_time_relations() train_loader = utils.make_data_loader(tr_texts, tr_labels, cfg.getint('model', 'batch_size'), None, 'train', utils.to_lstm_inputs) val_data = reldata.RelData(os.path.join(base, cfg.get('data', 'xmi_dir')), partition='dev', n_files=cfg.get('data', 'n_files')) val_texts, val_labels = val_data.event_time_relations() val_loader = utils.make_data_loader(val_texts, val_labels, cfg.getint('model', 'batch_size'), None, 'dev', utils.to_lstm_inputs) model = LstmClassifier() label_counts = torch.bincount(torch.IntTensor(tr_labels)) weights = len(tr_labels) / (2.0 * label_counts) train(model, train_loader, val_loader, weights) evaluate(model, val_loader, weights, suppress_output=False)
def main(): """Fine-tune bert""" train_data = reldata.RelData(os.path.join(base, cfg.get('data', 'xmi_dir')), partition='train') train_texts, train_labels = train_data.event_time_relations() train_texts = to_inputs(train_texts) print('train text shape:', train_texts['word_inputs'].shape) model = get_model() model.compile( optimizer=tf.keras.optimizers.Adam(learning_rate=cfg.getfloat( 'bert', 'lr'), epsilon=1e-08, clipnorm=1.0), loss=tf.keras.losses.SparseCategoricalCrossentropy(), metrics=[tf.keras.metrics.SparseCategoricalAccuracy('accuracy')]) model.fit(x=train_texts, y=train_labels, epochs=cfg.getint('bert', 'num_epochs'), batch_size=cfg.getint('bert', 'batch_size')) dev_data = reldata.RelData(os.path.join(base, cfg.get('data', 'xmi_dir')), partition='dev') dev_texts, dev_labels = dev_data.event_time_relations() dev_texts = to_inputs(dev_texts) dev_predictions = model.predict(dev_texts) dev_predictions = np.argmax(dev_predictions, axis=1) performance_metrics(dev_labels, dev_predictions)
def main(): """Fine-tune bert""" model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=3) if torch.cuda.is_available(): device = torch.device('cuda') model.cuda() else: device = torch.device('cpu') model.cpu() optimizer, scheduler = make_optimizer_and_scheduler(model) train_data = reldata.RelData(os.path.join(base, cfg.get('data', 'xmi_dir'))) train_loader = make_data_loader(train_data, RandomSampler) for epoch in trange(cfg.getint('bert', 'num_epochs'), desc='epoch'): model.train() train_loss, num_train_examples, num_train_steps = 0, 0, 0 for step, batch in enumerate(train_loader): batch = tuple(t.to(device) for t in batch) batch_inputs, batch_masks, batch_labels = batch optimizer.zero_grad() loss, logits = model(batch_inputs, attention_mask=batch_masks, labels=batch_labels) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) optimizer.step() scheduler.step() train_loss += loss.item() num_train_examples += batch_inputs.size(0) num_train_steps += 1 print('epoch: %d, loss: %.4f' % (epoch, train_loss / num_train_steps)) dev_data = reldata.RelData(os.path.join(base, cfg.get('data', 'xmi_dir')), partition='dev', xml_ref_dir=os.path.join( base, cfg.get('data', 'ref_xml_dir')), xml_out_dir=cfg.get('data', 'out_xml_dir')) dev_loader = make_data_loader(dev_data, sampler=SequentialSampler) predictions = evaluate(model, dev_loader, device)
def main(): """Fine-tune bert""" train_data = reldata.RelData(os.path.join(base, cfg.get('data', 'xmi_dir')), partition='train', n_files=cfg.get('data', 'n_files')) tr_texts, tr_labels = train_data.event_time_relations() train_loader = make_data_loader(tr_texts, tr_labels, RandomSampler) val_data = reldata.RelData(os.path.join(base, cfg.get('data', 'xmi_dir')), partition='dev', n_files=cfg.get('data', 'n_files')) val_texts, val_labels = val_data.event_time_relations() val_loader = make_data_loader(val_texts, val_labels, SequentialSampler) model = BertClassifier.from_pretrained('bert-base-uncased', num_labels=2) label_counts = torch.bincount(torch.IntTensor(tr_labels)) weights = len(tr_labels) / (2.0 * label_counts) print('class weights:', weights) train(model, train_loader, val_loader, weights) evaluate(model, val_loader, weights, suppress_output=False)