def train(train_dataset, model: BertForSequenceClassification, optimizer: AdamW, batch_size=batch_size): train_sampler = RandomSampler(train_dataset) train_loader = DataLoader(train_dataset, sampler=train_sampler, batch_size=batch_size) model.train() tr_loss = 0.0 tr_acc = 0 global_step = 0 if cuda: torch.cuda.empty_cache() for step, batch in tqdm(enumerate(train_loader)): # print(step) inputs = { 'input_ids': batch[1], 'token_type_ids': batch[2], 'attention_mask': batch[3], 'labels': batch[0] } outputs = model(**inputs) loss = outputs[0] # print(loss) logits = outputs[1] tr_loss += loss.item() model.zero_grad() loss.backward() optimizer.step() # 计算准确率 _, pred = logits.max(1) number_corr = (pred == batch[0].view(-1)).long().sum().item() tr_acc += number_corr global_step += 1 return tr_loss / global_step, tr_acc / len(train_dataset)
def train_model(device: torch.device, model: BertForSequenceClassification, settings: Settings, train_dataloader: DataLoader): settings.write_debug('Starting train hate speech model') optimizer = get_optimizer(settings.get_optimizer_name(), model.parameters(), settings) # Number of training epochs (authors recommend between 2 and 4) epochs = settings.get_num_training_epochs() # Total number of training steps is number of batches * number of epochs. total_steps = len(train_dataloader) * epochs # Create the learning rate scheduler. scheduler = get_linear_schedule_with_warmup( optimizer, num_warmup_steps=settings.get_num_warmup_steps( ), # Default value in run_glue.py num_training_steps=total_steps) # This training code is based on the `run_glue.py` script here: # https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128 # Set the seed value all over the place to make this reproducible. random_seed = settings.get_random_seed() random.seed(random_seed) np.random.seed(random_seed) torch.manual_seed(random_seed) torch.cuda.manual_seed_all(random_seed) # Store the average loss after each epoch so we can plot them. loss_values = [] # For each epoch... for epoch_i in range(0, epochs): # ======================================== # Training # ======================================== # Perform one full pass over the training set. settings.write_debug("") settings.write_debug('======== Epoch {:} / {:} ========'.format( epoch_i + 1, epochs)) settings.write_debug('Training...') # Measure how long the training epoch takes. t0 = time.time() # Reset the total loss for this epoch. total_loss = 0 # Put the model into training mode. Don't be mislead--the call to # `train` just changes the *mode*, it doesn't *perform* the training. # `dropout` and `batchnorm` layers behave differently during training # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch) model.train() # For each batch of training data... for step, batch in enumerate(train_dataloader): # Progress update every 40 batches. if step % 40 == 0 and not step == 0: # Calculate elapsed time in minutes. elapsed = format_time(time.time() - t0) # Report progress. settings.write_debug( ' Batch {:>5,} of {:>5,}. Elapsed: {:}.'.format( step, len(train_dataloader), elapsed)) # Unpack this training batch from our dataloader. # # As we unpack the batch, we'll also copy each tensor to the GPU using the # `to` method. # # `batch` contains three pytorch tensors: # [0]: input ids # [1]: attention masks # [2]: labels b_input_ids = batch[0].to(device) b_input_mask = batch[1].to(device) b_labels = batch[2].to(device) # Always clear any previously calculated gradients before performing a # backward pass. PyTorch doesn't do this automatically because # accumulating the gradients is "convenient while training RNNs". # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch) model.zero_grad() # Perform a forward pass (evaluate the model on this training batch). # This will return the loss (rather than the model output) because we # have provided the `labels`. # The documentation for this `model` function is here: # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels) # The call to `model` always returns a tuple, so we need to pull the # loss value out of the tuple. loss = outputs[0] # Accumulate the training loss over all of the batches so that we can # calculate the average loss at the end. `loss` is a Tensor containing a # single value; the `.item()` function just returns the Python value # from the tensor. total_loss += loss.item() # Perform a backward pass to calculate the gradients. loss.backward() # Clip the norm of the gradients to 1.0. # This is to help prevent the "exploding gradients" problem. torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) # Update parameters and take a step using the computed gradient. # The optimizer dictates the "update rule"--how the parameters are # modified based on their gradients, the learning rate, etc. optimizer.step() # Update the learning rate. scheduler.step() # Calculate the average loss over the training data. avg_train_loss = total_loss / len(train_dataloader) # Store the loss value for plotting the learning curve. loss_values.append(avg_train_loss) settings.write_debug("") settings.write_debug( " Average training loss: {0:.2f}".format(avg_train_loss)) settings.write_debug(" Training epcoh took: {:}".format( format_time(time.time() - t0))) settings.write_debug("") settings.write_debug("Training complete!") settings.write_debug('Finished train hate speech model')
def train_process(config, train_load, train_sampler, model_name): # load source bert weights model_config = BertConfig.from_pretrained( pretrained_model_name_or_path="../user_data/bert_source/{}_config.json" .format(model_name)) # model_config = BertConfig() model_config.vocab_size = len( pd.read_csv('../user_data/vocab', names=["score"])) model = BertForSequenceClassification(config=model_config) checkpoint = torch.load( '../user_data/save_bert/{}_checkpoint.pth.tar'.format(model_name), map_location=torch.device('cpu')) model.load_state_dict(checkpoint['status'], strict=False) print('***********load pretrained mlm {} weight*************'.format( model_name)) for param in model.parameters(): param.requires_grad = True # 4) 封装之前要把模型移到对应的gpu model = model.to(config.device) no_decay = ["bias", "LayerNorm.weight"] optimizer_grouped_parameters = [ { "params": [ p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay) ], "weight_decay": config.weight_decay, }, { "params": [ p for n, p in model.named_parameters() if any(nd in n for nd in no_decay) ], "weight_decay": 0.0 }, ] optimizer = AdamW(optimizer_grouped_parameters, lr=config.learning_rate) # t_total = len(train_load) * config.num_train_epochs # scheduler = get_linear_schedule_with_warmup( # optimizer, num_warmup_steps=t_total * config.warmup_proportion, num_training_steps=t_total # ) cudnn.benchmark = True if torch.cuda.device_count() > 1: print("Let's use", torch.cuda.device_count(), "GPUs!") # 5)封装 model = torch.nn.parallel.DistributedDataParallel( model, device_ids=[config.local_rank]) model.train() if config.fgm: fgm = FGM(model) for epoch in range(config.num_train_epochs): train_sampler.set_epoch(epoch) torch.cuda.empty_cache() for batch, (input_ids, token_type_ids, attention_mask, label) in enumerate(train_load): input_ids = input_ids.cuda(config.local_rank, non_blocking=True) attention_mask = attention_mask.cuda(config.local_rank, non_blocking=True) token_type_ids = token_type_ids.cuda(config.local_rank, non_blocking=True) label = label.cuda(config.local_rank, non_blocking=True) outputs = model(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, labels=label) loss = outputs.loss model.zero_grad() loss.backward() # torch.nn.utils.clip_grad_norm_(model.parameters(), config.max_grad_norm) if config.fgm: fgm.attack() # 在embedding上添加对抗扰动 loss_adv = model(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, labels=label).loss loss_adv.backward() # 反向传播,并在正常的grad基础上,累加对抗训练的梯度 fgm.restore() # 恢复embedding参数 optimizer.step() # scheduler.step() # dev_auc = model_evaluate(config, model, valid_load) # 同步各个进程的速度,计算分布式loss torch.distributed.barrier() # reduce_dev_auc = reduce_auc(dev_auc, config.nprocs).item() # if reduce_dev_auc > best_dev_auc: # best_dev_auc = reduce_dev_auc # is_best = True now = strftime("%Y-%m-%d %H:%M:%S", localtime()) msg = 'model_name:{},time:{},epoch:{}/{}' if config.local_rank in [0, -1]: print( msg.format(model_name, now, epoch + 1, config.num_train_epochs)) checkpoint = {"status": model.module.state_dict()} torch.save( checkpoint, '../user_data/save_model' + os.sep + '{}_checkpoint.pth.tar'.format(model_name)) del checkpoint torch.distributed.barrier()
class Classifier: """The Classifier""" ############################################# def __init__(self, train_batch_size=16, eval_batch_size=8, max_length=128, lr=2e-5, eps=1e-6, n_epochs=11): """ :param train_batch_size: (int) Training batch size :param eval_batch_size: (int) Batch size while using the `predict` method. :param max_length: (int) Maximum length for padding :param lr: (float) Learning rate :param eps: (float) Adam optimizer epsilon parameter :param n_epochs: (int) Number of epochs to train """ # model parameters self.train_batch_size = train_batch_size self.eval_batch_size = eval_batch_size self.max_length = max_length self.lr = lr self.eps = eps self.n_epochs = n_epochs # Information to be set or updated later self.trainset = None self.categories = None self.labels = None self.model = None # Tokenizer self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') # The model # # We first need to specify some configurations to the model configs = BertConfig.from_pretrained( 'bert-base-uncased', num_labels=3, type_vocab_size=8) # BERT configuration self.model = BertForSequenceClassification(configs) # We are changing the header classifier of the model (Which is initially a simple fully connect layer layer) clf = Net() self.model.classifier = clf self.model.to( device ) # putting the model on GPU if available otherwise device is CPU def preprocess(self, sentences): """ The preprocessing function :param sentences: List of all sentences to be given at once. :return: List of preprocessed sentences. """ preprocessed = [] for sentence in tqdm(sentences): assert isinstance(sentence, str) doc = nlp(str(sentence)) tokens = [] for token in doc: if (not token.is_punct) or (token.text not in [ ',', '-', '.', "'", '!' ]): # Some punctuations can be interesting for BERT tokens.append(token.text) tokens = (' '.join(tokens)).lower().replace(" '", "'") preprocessed.append(tokens) return preprocessed def question(self, category): """ Computes the questions corresponding to each category :param category: (str) The category/aspect :return: (str) computed question using the QA-M task """ assert category in self.categories if category == 'AMBIENCE#GENERAL': return "what do you think of the ambience of it ?" elif category == 'DRINKS#PRICES' or category == 'FOOD#PRICES' or category == 'RESTAURANT#PRICES': return "what do you think of the price of it ?" elif category == 'DRINKS#QUALITY' or category == 'FOOD#QUALITY': return "what do you think of the quality of it ?" elif category == 'DRINKS#STYLE_OPTIONS': return "what do you think of drinks ?" elif category == 'FOOD#STYLE_OPTIONS': return "what do you think of the food ?" elif category == 'LOCATION#GENERAL': return "what do you think of the location of it ?" elif category == 'RESTAURANT#GENERAL' or category == 'RESTAURANT#MISCELLANEOUS': return "what do you think of the restaurant ?" elif category == 'SERVICE#GENERAL': return "what do you think of the service of it ?" def train(self, trainfile): """Trains the classifier model on the training set stored in file trainfile""" # Loading the data and splitting up its information in lists print("\n Loading training data...") trainset = np.genfromtxt(trainfile, delimiter='\t', dtype=str, comments=None) self.trainset = trainset n = len(trainset) targets = trainset[:, 0] categories = trainset[:, 1] self.labels = list(Counter(targets).keys()) # label names self.categories = list(Counter(categories).keys()) # category names start_end = [[int(x) for x in w.split(':')] for w in trainset[:, 3]] # target words words_of_interest = [ trainset[:, 4][i][start_end[i][0]:start_end[i][1]] for i in range(n) ] # sentences to be classified sentences = [str(s) for s in trainset[:, 4]] # Preprocessing the text data print(" Preprocessing the text data...") sentences = self.preprocess(sentences) # Computing question sequences print(" Computing questions...") questions = [self.question(categories[i]) for i in tqdm(range(n))] # Tokenization attention_masks = [] input_ids = [] token_type_ids = [] labels = [] for word, question, answer in zip(words_of_interest, questions, sentences): encoded_dict = self.tokenizer.encode_plus( answer, question + ' ' + word.lower(), add_special_tokens=True, # Add '[CLS]' and '[SEP]' tokens max_length=self.max_length, # Pad & truncate all sequences pad_to_max_length=True, return_attention_mask=True, # Construct attention masks return_tensors='pt', # Return pytorch tensors. ) attention_masks.append(encoded_dict['attention_mask']) input_ids.append(encoded_dict['input_ids']) token_type_ids.append(encoded_dict['token_type_ids']) attention_masks = torch.cat(attention_masks, dim=0) input_ids = torch.cat(input_ids, dim=0) token_type_ids = torch.cat(token_type_ids, dim=0) # Converting polarities into integers (0: positive, 1: negative, 2: neutral) for target in targets: if target == 'positive': labels.append(0) elif target == 'negative': labels.append(1) elif target == 'neutral': labels.append(2) labels = torch.tensor(labels) # Pytorch data iterators train_data = TensorDataset(input_ids, attention_masks, token_type_ids, labels) train_sampler = RandomSampler(train_data) train_dataloader = DataLoader(train_data, batch_size=self.train_batch_size, sampler=train_sampler) # Optimizer and scheduler (we are using a linear scheduler without warm up) no_decay = ['bias', 'gamma', 'beta'] # These parameters are not going to be decreased optimizer_parameters = [{ 'params': [ p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay) ], 'weight_decay': 0.01 }, { 'params': [ p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay) ], 'weight_decay': 0.0 }] optimizer = AdamW(optimizer_parameters, lr=self.lr, eps=self.eps) total_steps = len(train_dataloader) * self.n_epochs scheduler = get_linear_schedule_with_warmup( optimizer, num_warmup_steps=0, num_training_steps=total_steps) # Training initial_t0 = time.time() for epoch in range(self.n_epochs): print('\n ======== Epoch %d / %d ========' % (epoch + 1, self.n_epochs)) print(' Training...\n') t0 = time.time() total_train_loss = 0 self.model.train() for step, batch in enumerate(train_dataloader): batch = tuple(t.to(device) for t in batch) input_ids_, input_mask_, segment_ids_, label_ids_ = batch self.model.zero_grad() loss, _ = self.model(input_ids_, token_type_ids=segment_ids_, attention_mask=input_mask_, labels=label_ids_) total_train_loss += loss.item() loss.backward() # clip gradient norm torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.0) optimizer.step() scheduler.step() avg_train_loss = total_train_loss / len(train_dataloader) training_time = format_time(time.time() - t0) # print(" Average training loss: {0:.2f}".format(avg_train_loss)) print(" Training epoch duration: {:}".format(training_time)) print(" Total training time: {:}".format( format_time(time.time() - initial_t0))) def predict(self, datafile): """Predicts class labels for the input instances in file 'datafile' Returns the list of predicted labels """ # Loading the data and splitting up its information in lists evalset = np.genfromtxt(datafile, delimiter='\t', dtype=str, comments=None) m = len(evalset) categories = evalset[:, 1] start_end = [[int(x) for x in w.split(':')] for w in evalset[:, 3]] # target words words_of_interest = [ evalset[:, 4][i][start_end[i][0]:start_end[i][1]] for i in range(m) ] # sentences to be classified sentences = [str(s) for s in evalset[:, 4]] # Preprocessing the text data print("\n Preprocessing the text data...") sentences = self.preprocess(sentences) # Computing question sequences print(" Computing questions...") questions = [self.question(categories[i]) for i in tqdm(range(m))] # Tokenization attention_masks = [] input_ids = [] token_type_ids = [] for word, question, answer in zip(words_of_interest, questions, sentences): encoded_dict = self.tokenizer.encode_plus( answer, question + ' ' + word.lower(), add_special_tokens=True, # Add '[CLS]' and '[SEP]' max_length=self.max_length, # Pad & truncate all sequences pad_to_max_length=True, return_attention_mask=True, # Construct attention masks return_tensors='pt', # Return pytorch tensors. ) attention_masks.append(encoded_dict['attention_mask']) input_ids.append(encoded_dict['input_ids']) token_type_ids.append(encoded_dict['token_type_ids']) attention_masks = torch.cat(attention_masks, dim=0) input_ids = torch.cat(input_ids, dim=0) token_type_ids = torch.cat(token_type_ids, dim=0) # Pytorch data iterators eval_data = TensorDataset(input_ids, attention_masks, token_type_ids) eval_sampler = SequentialSampler(eval_data) eval_dataloader = DataLoader(eval_data, batch_size=self.eval_batch_size, sampler=eval_sampler) # Prediction named_labels = [] self.model.eval() for batch in eval_dataloader: batch = tuple(t.to(device) for t in batch) input_ids, input_mask, segment_ids = batch with torch.no_grad(): logits = self.model(input_ids, token_type_ids=segment_ids, attention_mask=input_mask)[0] logits = softmax(logits, dim=-1) logits = logits.detach().cpu().numpy() outputs = np.argmax(logits, axis=1) # converting integer labels into named labels for label in outputs: if label == 0: named_labels.append('positive') elif label == 1: named_labels.append('negative') elif label == 2: named_labels.append('neutral') return np.array(named_labels)
def main(): parser = argparse.ArgumentParser() ## Required parameters parser.add_argument("--data_dir", default='/hdd/lujunyu/dataset/multi_turn_corpus/ubuntu/', type=str, required=False, help="The input data dir. Should contain the .tsv files (or other data files) for the task.") parser.add_argument("--task_name", default='ubuntu', type=str, required=False, help="The name of the task to train.") parser.add_argument("--output_dir", default='/hdd/lujunyu/model/chatbert/ubuntu_without_pretraining/', type=str, required=False, help="The output directory where the model checkpoints will be written.") ## Other parameters parser.add_argument("--init_model_name", default='bert-base-uncased', type=str, help="Initial checkpoint (usually from a pre-trained BERT model).") parser.add_argument("--do_lower_case", default=True, action='store_true', help="Whether to lower case the input text. True for uncased models, False for cased models.") parser.add_argument("--data_augmentation", default=False, action='store_true', help="Whether to use augmentation") parser.add_argument("--max_seq_length", default=256, type=int, help="The maximum total input sequence length after WordPiece tokenization. \n" "Sequences longer than this will be truncated, and sequences shorter \n" "than this will be padded.") parser.add_argument("--do_train", default=True, action='store_true', help="Whether to run training.") parser.add_argument("--do_test", default=True, action='store_true', help="Whether to run eval on the test set.") parser.add_argument("--train_batch_size", default=500, type=int, help="Total batch size for training.") parser.add_argument("--eval_batch_size", default=500, type=int, help="Total batch size for eval.") parser.add_argument("--learning_rate", default=3e-3, type=float, help="The initial learning rate for Adam.") parser.add_argument("--num_train_epochs", default=10.0, type=float, help="Total number of training epochs to perform.") parser.add_argument("--warmup_steps", default=0.0, type=float, help="Proportion of training to perform linear learning rate warmup for. " "E.g., 0.1 = 10%% of training.") parser.add_argument("--weight_decay", default=1e-3, type=float, help="weight_decay") parser.add_argument("--save_checkpoints_steps", default=8000, type=int, help="How often to save the model checkpoint.") parser.add_argument("--no_cuda", default=False, action='store_true', help="Whether not to use CUDA when available") parser.add_argument("--local_rank", type=int, default=-1, help="local_rank for distributed training on gpus") parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") parser.add_argument('--gradient_accumulation_steps', type=int, default=20, help="Number of updates steps to accumualte before performing a backward/update pass.") args = parser.parse_args() if args.local_rank == -1 or args.no_cuda: device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") n_gpu = torch.cuda.device_count() else: device = torch.device("cuda", args.local_rank) n_gpu = 1 # Initializes the distributed backend which will take care of sychronizing nodes/GPUs torch.distributed.init_process_group(backend='nccl') logger.info("device %s n_gpu %d distributed training %r", device, n_gpu, bool(args.local_rank != -1)) if args.gradient_accumulation_steps < 1: raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format( args.gradient_accumulation_steps)) args.train_batch_size = int(args.train_batch_size / args.gradient_accumulation_steps) random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if n_gpu > 0: torch.cuda.manual_seed_all(args.seed) if not args.do_train and not args.do_eval: raise ValueError("At least one of `do_train` or `do_eval` must be True.") bert_config = BertConfig.from_pretrained(args.init_model_name, num_labels=2) if args.max_seq_length > bert_config.max_position_embeddings: raise ValueError( "Cannot use sequence length {} because the BERT model was only trained up to sequence length {}".format( args.max_seq_length, bert_config.max_position_embeddings)) if os.path.exists(args.output_dir) and os.listdir(args.output_dir): if args.do_train: raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir)) else: os.makedirs(args.output_dir, exist_ok=True) tokenizer = BertTokenizer.from_pretrained(args.init_model_name, do_lower_case=args.do_lower_case) if args.data_augmentation: train_dataset = UbuntuDatasetForSP( file_path=os.path.join(args.data_dir, "train_augment_3.txt"), max_seq_length=args.max_seq_length, tokenizer=tokenizer ) else: train_dataset = UbuntuDatasetForSP( file_path=os.path.join(args.data_dir, "train.txt"), max_seq_length=args.max_seq_length, tokenizer=tokenizer ) eval_dataset = UbuntuDatasetForSP( file_path=os.path.join(args.data_dir, "valid.txt"), max_seq_length=args.max_seq_length, tokenizer=tokenizer ) train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=args.train_batch_size, sampler=RandomSampler(train_dataset), num_workers=4) eval_dataloader = torch.utils.data.DataLoader(eval_dataset, batch_size=args.eval_batch_size, sampler=SequentialSampler(eval_dataset), num_workers=4) model = BertForSequenceClassification(config=bert_config) model.to(device) num_train_steps = None if args.do_train: num_train_steps = int( len(train_dataset) / args.train_batch_size / args.gradient_accumulation_steps * args.num_train_epochs) # Prepare optimizer param_optimizer = list(model.named_parameters()) # remove pooler, which is not used thus it produce None grad that break apex param_optimizer = [n for n in param_optimizer] no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [{ 'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay}, { 'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}] optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate) scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=num_train_steps) else: optimizer = None scheduler = None if args.local_rank != -1: model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], output_device=args.local_rank) elif n_gpu > 1: model = torch.nn.DataParallel(model) global_step = 0 best_metric = 0.0 if args.do_train: logger.info("***** Running training *****") logger.info(" Num examples = %d", len(train_dataset)) logger.info(" Batch size = %d", args.train_batch_size) logger.info(" Num steps = %d", num_train_steps) model.train() for _ in trange(int(args.num_train_epochs), desc="Epoch"): tr_loss = 0 nb_tr_examples, nb_tr_steps = 0, 0 for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration")): batch = tuple(t.to(device) for t in batch) input_ids, input_mask, segment_ids, label_ids = batch loss, _ = model(input_ids, token_type_ids=segment_ids, attention_mask=input_mask, labels=label_ids) if n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu. if args.gradient_accumulation_steps > 1: loss = loss / args.gradient_accumulation_steps loss.backward() tr_loss += loss.item() nb_tr_examples += input_ids.size(0) nb_tr_steps += 1 if (step + 1) % args.gradient_accumulation_steps == 0: optimizer.step() # We have accumulated enought gradients scheduler.step() model.zero_grad() global_step += 1 if step % args.save_checkpoints_steps == 0: model.eval() f = open(os.path.join(args.output_dir, 'logits_dev.txt'), 'w') eval_loss = 0 nb_eval_steps, nb_eval_examples = 0, 0 logits_all = [] for input_ids, input_mask, segment_ids, label_ids in eval_dataloader: input_ids = input_ids.to(device) input_mask = input_mask.to(device) segment_ids = segment_ids.to(device) label_ids = label_ids.to(device) with torch.no_grad(): tmp_eval_loss, logits = model(input_ids, token_type_ids=segment_ids, attention_mask=input_mask, labels=label_ids) logits = logits.detach().cpu().numpy() logits_all.append(logits) label_ids = label_ids.cpu().numpy() for logit, label in zip(logits, label_ids): logit = '{},{}'.format(logit[0], logit[1]) f.write('_\t{}\t{}\n'.format(logit, label)) eval_loss += tmp_eval_loss.mean().item() nb_eval_examples += input_ids.size(0) nb_eval_steps += 1 f.close() logits_all = np.concatenate(logits_all,axis=0) eval_loss = eval_loss / nb_eval_steps result = evaluate(os.path.join(args.output_dir, 'logits_dev.txt')) result.update({'eval_loss': eval_loss}) output_eval_file = os.path.join(args.output_dir, "eval_results_dev.txt") with open(output_eval_file, "a") as writer: logger.info("***** Eval results *****") for key in sorted(result.keys()): logger.info(" %s = %s", key, str(result[key])) writer.write("%s = %s\n" % (key, str(result[key]))) ### Save the best checkpoint if best_metric < result['R10@1'] + result['R10@2']: try: ### Remove 'module' prefix when using DataParallel state_dict = model.module.state_dict() except AttributeError: state_dict = model.state_dict() torch.save(state_dict, os.path.join(args.output_dir, "model.pt")) best_metric = result['R10@1'] + result['R10@2'] logger.info('Saving the best model in {}'.format(os.path.join(args.output_dir, "model.pt"))) ### visualize bad cases of the best model # logger.info('Saving Bad cases...') # visualize_bad_cases( # logits=logits_all, # input_file_path=os.path.join(args.data_dir, 'valid.txt'), # output_file_path=os.path.join(args.output_dir, 'valid_bad_cases.txt') # ) model.train()
total_train_loss = 0 model.train() for step, batch in enumerate(train_dataloader): if step % 40 == 0 and not step == 0: elapsed = format_time(time.time() - t0) print(' Batch {:>5,} of {:>5,}. Elapsed: {:}.'.format( step, len(train_dataloader), elapsed)) b_input_ids = batch[0].to(device) b_input_mask = batch[1].to(device) b_labels = batch[2].to(device) model.zero_grad() loss, logits = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels) total_train_loss += loss.item() loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) optimizer.step() scheduler.step() avg_train_loss = total_train_loss / len(train_dataloader)
def train_classifier(model: BertForSequenceClassification, dataset: TensorDataset, validation_ratio: float, batch_size: int, freeze_embeddings_layer: bool, freeze_encoder_layers: int, epochs: int) -> (BertForSequenceClassification, list): device = select_device() train_size = int(validation_ratio * len(dataset)) val_size = len(dataset) - train_size train_dataset, val_dataset = random_split(dataset, [train_size, val_size]) train_dataloader = DataLoader(train_dataset, sampler=RandomSampler(train_dataset), batch_size=batch_size) validation_dataloader = DataLoader(val_dataset, sampler=SequentialSampler(val_dataset), batch_size=batch_size) modules = [] if freeze_embeddings_layer: modules.append(model.bert.embeddings) for i in range(freeze_encoder_layers): modules.append(model.bert.encoder.layer[i]) for module in modules: for param in module.parameters(): param.requires_grad = False model.to(device) optimizer = AdamW(filter(lambda p: p.requires_grad, model.parameters()), lr=5e-5) total_steps = len(train_dataloader) * epochs scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0, num_training_steps=total_steps) training_stats = [] total_t0 = time.time() for epoch_i in range(0, epochs): print("") print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs)) print('Training...') t0 = time.time() total_train_loss = 0 model.train() for step, batch in enumerate(train_dataloader): if step % 40 == 0 and not step == 0: elapsed = format_time(time.time() - t0) print(' Batch {:>5,} of {:>5,}. Elapsed: {:}.'.format( step, len(train_dataloader), elapsed)) b_input_ids = batch[0].to(device) b_input_mask = batch[1].to(device) b_labels = batch[2].to(device) model.zero_grad() outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels) loss = outputs.loss logits = outputs.logits total_train_loss += loss.item() loss.backward() # Clip the norm of the gradients to 1.0. # This is to help prevent the "exploding gradients" problem. torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) optimizer.step() scheduler.step() avg_train_loss = total_train_loss / len(train_dataloader) training_time = format_time(time.time() - t0) print("") print(" Average training loss: {0:.2f}".format(avg_train_loss)) print(" Training epcoh took: {:}".format(training_time)) print("") print("Running Validation...") t0 = time.time() model.eval() total_eval_accuracy = 0 total_eval_loss = 0 nb_eval_steps = 0 for batch in validation_dataloader: b_input_ids = batch[0].to(device) b_input_mask = batch[1].to(device) b_labels = batch[2].to(device) with torch.no_grad(): outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels) loss = outputs.loss logits = outputs.logits total_eval_loss += loss.item() logits = logits.detach().cpu().numpy() label_ids = b_labels.cpu().numpy() total_eval_accuracy += flat_accuracy(logits, label_ids) avg_val_accuracy = total_eval_accuracy / len(validation_dataloader) print(" Accuracy: {0:.2f}".format(avg_val_accuracy)) avg_val_loss = total_eval_loss / len(validation_dataloader) validation_time = format_time(time.time() - t0) print(" Validation Loss: {0:.2f}".format(avg_val_loss)) print(" Validation took: {:}".format(validation_time)) training_stats.append({ 'epoch': epoch_i + 1, 'Training Loss': avg_train_loss, 'Valid. Loss': avg_val_loss, 'Valid. Accur.': avg_val_accuracy, 'Training Time': training_time, 'Validation Time': validation_time }) print("") print("Training complete!") print("Total training took {:} (h:mm:ss)".format( format_time(time.time() - total_t0))) return model, training_stats
def train_and_test(): # prepare data fileNameList = glob.glob( 'C:/YYQ/PGproject/PreProcessing/processed_features_facenet/*.pkl') # print(fileNameList) # basic features # text-list and tf-idf text_list = [] labels = [] visual_features = [] audio_features = [] for file_name in fileNameList: data_point = pkl.load(open(file_name, 'rb')) clip_name, label, transcription, smoothed_seq = data_point[ 0], data_point[1], data_point[2], data_point[3] # print(label, transcription) # continue labels.append(label) text_list.append(transcription) # average visual features # visual_seq = np.stack([w['landmark_feature'] for w in smoothed_seq], axis=0) visual_seq = np.stack( [w['facenet_feature'].squeeze() for w in smoothed_seq], axis=0) # visual_seq = scale(visual_seq) # visual_seq = visual_seq - np.mean(visual_seq, axis=0) # print(visual_seq.shape) # visual_mean = np.mean(visual_seq, axis=0) visual_features.append(visual_seq) # average audio features audio_seq = np.stack([w['audio_grp'] for w in smoothed_seq], axis=0) # audio_seq = scale(audio_seq) # print(audio_seq.shape) # audio_mean = np.mean(audio_seq, axis=0) audio_features.append(audio_seq) # exit() print(text_list) lens = [len(a.split()) for a in text_list] print(min(lens), max(lens)) exit() tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') pg = tokenizer.batch_encode_plus(text_list, max_length=128, pad_to_max_length=True) '''print(len(pg)) for k in pg.keys(): print(k, len(pg[k]), [len(kk) for kk in pg[k]])''' x = pg['input_ids'] token_type_ids = pg['token_type_ids'] attention_mask = pg['attention_mask'] '''for xx in x: print(xx)''' x, token_type_ids, attention_mask = np.array(x), np.array( token_type_ids), np.array(attention_mask) labels = np.array(labels) skf = StratifiedKFold(n_splits=5) cv5_ids = list(skf.split(x, labels)) sp = cv5_ids[0] train_l, train_labels = x[sp[0]], labels[sp[0]] # train_data, train_labels = sm.fit_sample(train_data, train_labels) test_l, test_labels = x[sp[1]], labels[sp[1]] print(train_l.shape) train_token_type_ids, test_token_type_ids, train_attention_mask, test_attention_mask = token_type_ids[sp[0]], \ token_type_ids[sp[1]], attention_mask[sp[0]], attention_mask[sp[1]] # shuffle training data for batch reading n_train = len(train_l) n_eval = len(test_l) perm = np.random.permutation(n_train) train_l = train_l[perm] train_labels = np.array(train_labels)[perm] train_token_type_ids, train_attention_mask = train_token_type_ids[ perm], train_attention_mask[perm] train_l, test_l, train_labels, test_labels, train_token_type_ids, test_token_type_ids = torch.LongTensor(train_l), \ torch.LongTensor(test_l), \ torch.LongTensor(train_labels), \ torch.LongTensor(test_labels), \ torch.LongTensor(train_token_type_ids), \ torch.LongTensor(test_token_type_ids) train_attention_mask, test_attention_mask = torch.FloatTensor(train_attention_mask), \ torch.FloatTensor(test_attention_mask) # model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=3).to('cuda') config = BertConfig.from_pretrained('bert-base-uncased', num_labels=3) model = BertForSequenceClassification(config).to('cuda') # print(model(train_l[:32], token_type_ids=train_token_type_ids[:32], attention_mask=train_attention_mask[:32], labels=train_labels[:32])[1]) eval_every = 5 batch_size = 32 test_batch_size = 8 max_epochs = 500 t_total = math.ceil(n_train / batch_size) * max_epochs lr = 2e-5 epsilon = 1e-8 max_grad_norm = 1.0 weight_decay = 0.0 optimizer, scheduler = get_optimizers(model, learning_rate=lr, adam_epsilon=epsilon, weight_decay=weight_decay, num_training_steps=t_total) # loss_fn = torch.nn.CrossEntropyLoss().cuda() model.train() model.zero_grad() for ep in range(max_epochs): idx = 0 avg_loss = 0 n_batch = 0 model.train() while idx < n_train: optimizer.zero_grad() batch_l = train_l[idx:(idx + batch_size)].to('cuda') batch_ty = train_token_type_ids[idx:(idx + batch_size)].to('cuda') batch_am = train_attention_mask[idx:(idx + batch_size)].to('cuda') ans = train_labels[idx:(idx + batch_size)].to('cuda') idx += batch_size preds = model(input_ids=batch_l, token_type_ids=batch_ty, attention_mask=batch_am, labels=ans) loss = preds[0] # print(preds, ans) loss.backward() # print(loss.data.cpu().numpy()) avg_loss += loss.data.cpu().numpy() n_batch += 1. torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm) optimizer.step() scheduler.step() model.zero_grad() avg_loss = avg_loss / n_batch print("epoch: %d avg_loss: %f" % (ep + 1, avg_loss)) del batch_l, batch_ty, batch_am torch.cuda.empty_cache() # time.sleep(20) if ep % eval_every == 0: idx = 0 model.eval() eval_preds = np.array([]) while idx < n_eval: test_batch_l = test_l[idx:(idx + test_batch_size)].to('cuda') test_batch_ty = test_token_type_ids[idx:( idx + test_batch_size)].to('cuda') test_batch_am = test_attention_mask[idx:( idx + test_batch_size)].to('cuda') test_ans = test_labels[idx:(idx + test_batch_size)].to('cuda') # time.sleep(20) # exit() test_pred = model(input_ids=test_batch_l, token_type_ids=test_batch_ty, attention_mask=test_batch_am, labels=test_ans) scores = test_pred[1] _, batch_eval_preds = scores.data.cpu().max(1) eval_preds = np.concatenate((eval_preds, batch_eval_preds), axis=-1) idx += test_batch_size # metrics precison, recall, fscore, support = precision_recall_fscore_support( test_labels.cpu().numpy(), eval_preds, labels=[0, 1, 2], average=None) '''scores = model(train_data, train_lens) _, train_preds = scores.data.cpu().max(1) print("training set: %f" % (float(sum(train_preds.numpy() == train_labels.cpu().numpy())) / len(train_preds.numpy()))) print(eval_preds.numpy())''' print( float(sum(eval_preds == test_labels.cpu().numpy())) / len(eval_preds)) print(precison, recall, fscore, support)