def __init__(self, model, num_steps, num_classes=2): super(GPT2Classifier, self).__init__() self.tokenizer = GPT2Tokenizer.from_pretrained(model) self.tokenizer.pad_token = self.tokenizer.eos_token self.encoder = GPT2ForSequenceClassification.from_pretrained(model) self.encoder.config.pad_token_id = self.tokenizer.eos_token_id self.num_steps = num_steps
def makeUnilabelModel(self, modelName, num_labels=10, root='', **kwargs): if modelName == 'distilbert-base-uncased': tokenizer = DistilBertTokenizerFast.from_pretrained( 'distilbert-base-uncased') model = DistilBertForSequenceClassification.from_pretrained( root + "distilbert-base-uncased", num_labels=num_labels, **kwargs) if modelName == 'gpt2': tokenizer = GPT2TokenizerFast.from_pretrained('gpt2') tokenizer.add_special_tokens({'pad_token': '[PAD]'}) model = GPT2ForSequenceClassification.from_pretrained( root + "gpt2", num_labels=num_labels, **kwargs) model.resize_token_embeddings(len(tokenizer)) # add padding token model.config.pad_token_id = tokenizer('[PAD]').input_ids[0] if modelName == 'bertweet': tokenizer = AutoTokenizer.from_pretrained('vinai/bertweet-base') model = AutoModelForSequenceClassification.from_pretrained( root + "vinai/bertweet-base", num_labels=num_labels, **kwargs) if modelName == 'distilroberta-base': tokenizer = AutoTokenizer.from_pretrained('distilroberta-base') model = AutoModelForSequenceClassification.from_pretrained( root + "distilroberta-base", num_labels=num_labels, **kwargs) if modelName == 'lstm': tokenizer = AutoTokenizer.from_pretrained( 'distilbert-base-uncased') model = LSTMCclassifier(128, 64, 2, tokenizer.vocab_size, num_labels) return tokenizer, model
def __init__(self, conf): super(GPT2Classifier, self).__init__() self.conf = conf self.padding = 0 self.gpt_path = getattr(conf, 'bert_path', None) pretrain_name = 'gpt2' if self.gpt_path: pretrain_name = self.gpt_path print('GPT Model from {}'.format(pretrain_name)) self.gpt = GPT2ForSequenceClassification.from_pretrained( pretrain_name, num_labels=conf.class_num) self.gpt.config.pad_token_id = self.gpt.config.eos_token_id self.drop = nn.Dropout(p=0.3)
def __init__(self): # Look for gpu to use. Will use `cpu` by default if no gpu found. self.device = torch.device( 'cuda' if torch.cuda.is_available() else 'cpu') # device = torch.device('cpu') print("Device: ", self.device) _ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) model_path = os.path.join(_ROOT, "Models", "GPT2_Model", "model") self.tokenizer = GPT2Tokenizer.from_pretrained( pretrained_model_name_or_path=model_path) self.gpt_model = GPT2ForSequenceClassification.from_pretrained( pretrained_model_name_or_path=model_path) self.gpt_model.eval()
from transformers import GPT2Tokenizer, GPT2ForSequenceClassification import torch tokenizer = GPT2Tokenizer.from_pretrained('microsoft/dialogrpt') model = GPT2ForSequenceClassification.from_pretrained('microsoft/dialogrpt') inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 outputs = model(**inputs, labels=labels) loss = outputs.loss logits = outputs.logits
validation_dataloader = torch.load(val_loader_fn) train_dataloader = torch.load(train_loader_fn) else: validation_dataloader, train_dataloader = create_train_val_loaders( config['dataset']['train'], val_loader_fn, train_loader_fn, tokenizer, batch_size, max_length) if os.path.exists(test_loader_fn) and (not build_new_dataloaders): pass else: create_test_loader(config['dataset']['test'], test_loader_fn, tokenizer, batch_size, max_length) # import pretrained model if model_name == "gpt2": model = GPT2ForSequenceClassification.from_pretrained( "gpt2", num_labels=NUM_LABELS) else: model = AutoModelForSequenceClassification.from_pretrained( config['model'][model_name], num_labels=NUM_LABELS) if cpu: # Distributor = torch.nn.parallel.DistributedDataParallelCPU # import torch.distributed as dist # rank=1 # world_size=12 # dist.init_process_group("gloo", world_size=world_size,rank=-1, store= None) # parallel_model = Distributor(model) parallel_model = model else: parallel_model = torch.nn.DataParallel(model) # Encapsulate the model parallel_model.cuda()
) # For validation the order doesn't matter, so we'll just read them sequentially. validation_dataloader = DataLoader( val_dataset, # The validation samples. sampler=SequentialSampler(val_dataset), # Pull out batches sequentially. batch_size=batch_size # Evaluate with this batch size. ) # Load GPT2ForSequenceClassification, the pretrained GPT2 model with a single # linear classification layer on top. model_config = GPT2Config.from_pretrained('gpt2', num_labels=2) # Get the actual model. print('Loading model...') model = GPT2ForSequenceClassification.from_pretrained('gpt2', config=model_config) # resize model embedding to match new tokenizer model.resize_token_embeddings(len(tokenizer)) # fix model padding token id model.config.pad_token_id = model.config.eos_token_id # Run this model on GPU. model.cuda() optimizer = AdamW( model.parameters(), lr= 2e-5, # args.learning_rate - default is 5e-5, paper recommends 5e-5/3e-5/2e-5 eps=1e-8 # args.adam_epsilon - default is 1e-8.
import numpy as np start_debugger_on_exception() train_dataset = DataSetBert(data_file= './data/data_train/train.csv') val_dataset = DataSetBert(data_file= './data/data_train/val.csv') test_dataset = DataSetBert(data_file= './data/data_train/test.csv') from torch.utils.data import DataLoader device = torch.device('cuda:1') train_dataloader = DataLoader(train_dataset, batch_size=11, shuffle=True) val_dataloader = DataLoader(val_dataset, batch_size=11, shuffle=True) test_dataloader = DataLoader(test_dataset, batch_size=11, shuffle=True) from transformers import BertTokenizer tokenizer = BertTokenizer.from_pretrained('uer/gpt2-chinese-cluecorpussmall') tokenizer.add_special_tokens({"additional_special_tokens": ["[P]"]}) model_config = GPT2Config.from_pretrained(pretrained_model_name_or_path='uer/gpt2-chinese-cluecorpussmall', num_labels=2) model_config.num_labels = 15 model =GPT2ForSequenceClassification.from_pretrained('uer/gpt2-chinese-cluecorpussmall',config = model_config) model.resize_token_embeddings(len(tokenizer)) model.config.pad_token_id = model.config.eos_token_id model.config.n_positions = 1024 assert model.config.num_labels == 15 model.to(device) model.train() model.to(device) import pdb;pdb.set_trace() from transformers import AdamW optimizer = AdamW(model.parameters(), lr=1e-5) no_decay = ['bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [ {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01}, {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ]
# Get model configuration. print('Loading configuraiton...') model_config = GPT2Config.from_pretrained(pretrained_model_name_or_path=model_name_or_path, num_labels=n_labels) # Get model's tokenizer. print('Loading tokenizer...') tokenizer = GPT2Tokenizer.from_pretrained(pretrained_model_name_or_path=model_name_or_path) # default to left padding tokenizer.padding_side = "left" # Define PAD Token = EOS Token = 50256 tokenizer.pad_token = tokenizer.eos_token # Get the actual model. print('Loading model...') model = GPT2ForSequenceClassification.from_pretrained(pretrained_model_name_or_path=model_name_or_path, config=model_config) # resize model embedding to match new tokenizer model.resize_token_embeddings(len(tokenizer)) # fix model padding token id model.config.pad_token_id = model.config.eos_token_id # Load model to defined device. model.to(device) print('Model loaded to `%s`'%device) """# Getting and training model""" train_dataset = MovieReviewsDataset(use_tokenizer = tokenizer, labels = label_train, texts = text_train) valid_dataset = MovieReviewsDataset(use_tokenizer = tokenizer, labels = label_test, texts = text_test)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--train', type=str, default='data/disaster_response_messages_training.csv') parser.add_argument('--test', type=str, default='data/disaster_response_messages_test.csv') parser.add_argument( '--validation', type=str, default='data/disaster_response_messages_validation.csv') parser.add_argument('--epoch', type=str, default='10') parser.add_argument('--model', type=str, default='bert', choices=['bert', 'bart', 'gpt2', 'roberta', 'xlnet']) args = parser.parse_args() EPOCH = int(args.epoch) model_name = args.model # create data loader for training and validation if model_name == 'bert': train_set = BertDataset(args.train) val_set = BertDataset(args.validation) test_set = BertDataset(args.test) elif model_name == 'bart': train_set = BartDataset(args.train) val_set = BartDataset(args.validation) test_set = BartDataset(args.test) elif model_name == 'gpt2': train_set = GPT2Dataset(args.train) val_set = GPT2Dataset(args.validation) test_set = GPT2Dataset(args.test) elif model_name == 'roberta': train_set = RobertaDataset(args.train) val_set = RobertaDataset(args.validation) test_set = RobertaDataset(args.test) elif model_name == 'xlnet': train_set = XLNetDataset(args.train) val_set = XLNetDataset(args.validation) test_set = XLNetDataset(args.test) train_loader = DataLoader(train_set, batch_size=20, shuffle=True) val_loader = DataLoader(val_set, batch_size=20, shuffle=False) test_loader = DataLoader(test_set, batch_size=20, shuffle=False) print('Data Loaded.') if model_name == 'bert': model = BertForSequenceClassification.from_pretrained( 'bert-base-uncased', num_labels=2) elif model_name == 'gpt2': model = GPT2ForSequenceClassification.from_pretrained('gpt2', num_labels=2) model.config.pad_token_id = model.config.eos_token_id elif model_name == 'bart': model = BartForSequenceClassification.from_pretrained( 'facebook/bart-base', num_labels=2) elif model_name == 'roberta': model = RobertaForSequenceClassification.from_pretrained( 'roberta-base', num_labels=2) elif model_name == 'xlnet': model = XLNetForSequenceClassification.from_pretrained( 'xlnet-base-cased', num_labels=2) optimizer = AdamW(model.parameters(), lr=2e-5, correct_bias=False) total_steps = len(train_loader) * EPOCH scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0, num_training_steps=total_steps) criterion = nn.CrossEntropyLoss() print('\nModel: ', model_name, '\tEpochs: ', EPOCH) epoch_loss = [] epoch_val_acc = [] for epoch in range(EPOCH): tqdm.write('Epoch: {}'.format(epoch + 1)) loss = train(model, train_loader, criterion, optimizer, scheduler) epoch_loss.append(loss) val_acc = val(model, val_loader) epoch_val_acc.append(val_acc) torch.save(model, model_name + '/' + model_name + '_model.pt') # model = torch.load(model_name+'_model.pt') tqdm.write('\nFinal test...') test_result = test(model, test_loader) with open(model_name + '/' + model_name + '_loss.p', 'wb') as f: pickle.dump(epoch_loss, f) with open(model_name + '/' + model_name + '_val_accuracy.p', 'wb') as f: pickle.dump(epoch_val_acc, f) with open(model_name + '/' + model_name + '_test_result.p', 'wb') as f: pickle.dump(test_result, f)
labels=targets) outputs, loss = outputs["logits"], outputs["loss"] test_acc += accuracy_(outputs, targets).item() print(f"Test Acc: {test_acc / len(test_loader)}") # как мы и ожидали, получили тот же результат, но процесс тренировки более гладкий (эпох можно было выбрать 4, на 5 уже оверфитится) # ## Fine-tuning # # Теперь другой подход: загрузим модель, которая обучалась решать задачу Language Modeling. Посмотрим, получим ли мы прирост в качестве. # In[7]: model_1 = GPT2ForSequenceClassification.from_pretrained( "distilgpt2", output_attentions=True, pad_token_id=tokenizer.eos_token_id, num_labels=8).to(device) # In[13]: from transformers import AdamW, get_linear_schedule_with_warmup # lr = 1e-5 # Предполагаемый learning rate. Он может быть больше или меньше :) criterion = nn.CrossEntropyLoss( ) # в принципе лишнее, внутри GPT2ForSequenceClassification уже он есть optimizer = AdamW(model_1.parameters(), lr=2e-5, eps=1e-8) num_epochs = 10 total_steps = len(train_loader) * num_epochs scheduler = get_linear_schedule_with_warmup(
tokenizer = GPT2Tokenizer.from_pretrained( pretrained_model_name_or_path=model_name_or_path) # Creating the tokenizer is pretty standard when using the Transformers library. # After creating the tokenizer it is critical for this tutorial to set padding to the left tokenizer.padding_side = "left" # and initialize the padding token to tokenizer.eos_token which is the GPT2's original end of sequence token. # This is the most essential part of this tutorial since GPT2 uses the last token for prediction so we need to pad to the left. # default to left padding tokenizer.padding_side = "left" # Define PAD Token = EOS Token = 50256 tokenizer.pad_token = tokenizer.eos_token # Get the actual model. print('Loading model...') model = GPT2ForSequenceClassification.from_pretrained( pretrained_model_name_or_path=model_name_or_path, config=model_config, cache_dir="/home/jovyan/data-vol-1/gpt2/models") print( f"number of parameters: {sum(p.numel() for p in model.parameters() if p.requires_grad)}" ) # resize model embedding to match new tokenizer model.resize_token_embeddings(len(tokenizer)) # fix model padding token id model.config.pad_token_id = model.config.eos_token_id # Load model to defined device. model.to(device) print('Model loaded to `%s`' % device)