def main(): parser = argparse.ArgumentParser() ## Required parameters parser.add_argument("--data_dir", default=None, type=str, required=True, help="The input data dir. Should contain the .tsv files (or other data files) for the task.") parser.add_argument("--bert_config_file", default=None, type=str, required=True, help="The config json file corresponding to the pre-trained BERT model. \n" "This specifies the model architecture.") parser.add_argument("--vocab_file", default=None, type=str, required=True, help="The vocabulary file that the BERT model was trained on.") parser.add_argument("--output_dir", default=None, type=str, required=True, help="The output directory where the model checkpoints will be written.") ## Other parameters parser.add_argument("--init_checkpoint", default=None, type=str, help="Initial checkpoint (usually from a pre-trained BERT model).") parser.add_argument("--do_lower_case", default=False, action='store_true', help="Whether to lower case the input text. True for uncased models, False for cased models.") parser.add_argument("--max_seq_length", default=128, type=int, help="The maximum total input sequence length after WordPiece tokenization. \n" "Sequences longer than this will be truncated, and sequences shorter \n" "than this will be padded.") parser.add_argument("--do_train", default=False, action='store_true', help="Whether to run training.") parser.add_argument("--do_eval", default=False, action='store_true', help="Whether to run eval on the dev set.") parser.add_argument("--train_batch_size", default=32, type=int, help="Total batch size for training.") parser.add_argument("--eval_batch_size", default=8, type=int, help="Total batch size for eval.") parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument("--num_train_epochs", default=3.0, type=float, help="Total number of training epochs to perform.") parser.add_argument("--warmup_proportion", default=0.1, type=float, help="Proportion of training to perform linear learning rate warmup for. " "E.g., 0.1 = 10%% of training.") parser.add_argument("--save_checkpoints_steps", default=1000, type=int, help="How often to save the model checkpoint.") parser.add_argument("--no_cuda", default=False, action='store_true', help="Whether not to use CUDA when available") parser.add_argument("--local_rank", type=int, default=-1, help="local_rank for distributed training on gpus") parser.add_argument("--gpu_ids", type=str, default="0", help="select one gpu to use") parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") parser.add_argument('--gradient_accumulation_steps', type=int, default=1, help="Number of updates steps to accumualte before performing a backward/update pass.") parser.add_argument('--optimize_on_cpu', default=False,action='store_true', help="Whether to perform optimization and keep the optimizer averages on CPU") parser.add_argument('--fp16', default=False, action='store_true', help="Whether to use 16-bit float precision instead of 32-bit") parser.add_argument('--loss_scale', type=float, default=128, help='Loss scaling, positive power of 2 values can improve fp16 convergence.') ################## atec parser.add_argument('--do_submit', default=False, action='store_true', help="submit to results to atec cloud") parser.add_argument('--train_devset', default=False, action='store_true', help="") parser.add_argument("--test_in_file", default=None, type=str, help="") parser.add_argument("--test_out_file", default=None, type=str, help="") args = parser.parse_args() if args.local_rank == -1 and not args.no_cuda: device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") #device = torch.device("cuda", args.gpu_id) n_gpu = len(args.gpu_ids.split(','))#torch.cuda.device_count() elif args.no_cuda: device = torch.device('cpu') n_gpu = 0 else: device = torch.device("cuda", args.local_rank) n_gpu = 1 # Initializes the distributed backend which will take care of sychronizing nodes/GPUs torch.distributed.init_process_group(backend='nccl') if args.fp16: logger.info("16-bits training currently not supported in distributed training") args.fp16 = False # (see https://github.com/pytorch/pytorch/pull/13496) logger.info("device %s n_gpu %d distributed training %r", device, n_gpu, bool(args.local_rank != -1)) if args.gradient_accumulation_steps < 1: raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format( args.gradient_accumulation_steps)) args.train_batch_size = int(args.train_batch_size / args.gradient_accumulation_steps) random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if n_gpu > 0: torch.cuda.manual_seed_all(args.seed) #if not args.do_train and not args.do_eval: # raise ValueError("At least one of `do_train` or `do_eval` must be True.") bert_config = BertConfig.from_json_file(args.bert_config_file) if args.max_seq_length > bert_config.max_position_embeddings: raise ValueError( "Cannot use sequence length {} because the BERT model was only trained up to sequence length {}".format( args.max_seq_length, bert_config.max_position_embeddings)) #if os.path.exists(args.output_dir) and os.listdir(args.output_dir): # raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir)) if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) #, exist_ok=True) processor = AtecProcessor() label_list = processor.get_labels() tokenizer = tokenization.FullTokenizer( vocab_file=args.vocab_file, do_lower_case=args.do_lower_case) train_examples = None num_train_steps = None if args.do_train: train_examples = processor.get_train_examples(args.data_dir) if args.train_devset: eval_examples = processor.get_dev_examples(args.data_dir) train_examples += eval_examples num_train_steps = int( len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps * args.num_train_epochs) # Prepare model logger.info('build model') model = BertForSequenceClassification(bert_config, len(label_list)) #model = BertSiameseModel(bert_config, len(label_list)) if args.init_checkpoint is not None: try: # just model.bert model.bert.load_state_dict(torch.load(args.init_checkpoint, map_location='cpu')) except RuntimeError as e: # all model import re new_state_dict = collections.OrderedDict() state_dict = torch.load(args.init_checkpoint, map_location='cpu') for key in state_dict.keys(): new_key = re.sub("module\.", "", key) new_state_dict[new_key] = state_dict[key] model.load_state_dict(new_state_dict) if args.fp16: model.half() model.to(device) if args.local_rank != -1: model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], output_device=args.local_rank) elif n_gpu > 1: model = torch.nn.DataParallel(model, device_ids=[int(x) for x in args.gpu_ids.split(',')]) global_step = 0 tr_loss=None if args.do_train: # Prepare optimizer if args.fp16: param_optimizer = [(n, param.clone().detach().to('cpu').float().requires_grad_()) \ for n, param in model.named_parameters()] elif args.optimize_on_cpu: param_optimizer = [(n, param.clone().detach().to('cpu').requires_grad_()) \ for n, param in model.named_parameters()] else: param_optimizer = list(model.named_parameters()) no_decay = ['bias', 'gamma', 'beta'] optimizer_grouped_parameters = [ {'params': [p for n, p in param_optimizer if n not in no_decay], 'weight_decay_rate': 0.01}, {'params': [p for n, p in param_optimizer if n in no_decay], 'weight_decay_rate': 0.0} ] optimizer = BERTAdam(optimizer_grouped_parameters, lr=args.learning_rate, warmup=args.warmup_proportion, t_total=num_train_steps) from tqdm import tqdm, trange from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler from torch.utils.data.distributed import DistributedSampler train_features = convert_examples_to_features( train_examples, label_list, args.max_seq_length, tokenizer) # train_features = convert_examples_to_siamese_features( # train_examples, label_list, args.max_seq_length, tokenizer) logger.info("***** Running training *****") logger.info(" Num examples = %d", len(train_examples)) logger.info(" Batch size = %d", args.train_batch_size) logger.info(" Num steps = %d", num_train_steps) all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long) all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long) train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) # all_tokens_a = torch.tensor([f.tokens_a for f in train_features], dtype=torch.long) # all_types_a = torch.tensor([f.types_a for f in train_features], dtype=torch.long) # all_mask_a = torch.tensor([f.mask_a for f in train_features], dtype=torch.long) # all_tokens_b = torch.tensor([f.tokens_b for f in train_features], dtype=torch.long) # all_types_b = torch.tensor([f.types_b for f in train_features], dtype=torch.long) # all_mask_b = torch.tensor([f.mask_b for f in train_features], dtype=torch.long) # all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long) # train_data = TensorDataset(all_tokens_a, all_types_a, all_mask_a, # all_tokens_b, all_types_b, all_mask_b, all_label_ids) if args.local_rank == -1: train_sampler = RandomSampler(train_data) else: train_sampler = DistributedSampler(train_data) train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size) model.train() for _ in trange(int(args.num_train_epochs), desc="Epoch"): tr_loss = 0 nb_tr_examples, nb_tr_steps = 0, 0 with tqdm(train_dataloader, desc="Iteration") as pbar: for step, batch in enumerate(pbar): batch = tuple(t.to(device) for t in batch) input_ids, input_mask, segment_ids, label_ids = batch loss, logits = model(input_ids, segment_ids, input_mask, label_ids) # tokens_a, types_a, mask_a, tokens_b, types_b, mask_b, label_ids = batch # loss, logits = model(tokens_a, types_a, mask_a, tokens_b, types_b, # mask_b, label_ids) if n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu. if args.fp16 and args.loss_scale != 1.0: # rescale loss for fp16 training # see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html loss = loss * args.loss_scale if args.gradient_accumulation_steps > 1: loss = loss / args.gradient_accumulation_steps loss.backward() tr_loss += loss.item() preds = th.argmax(logits, dim=1) acc = accuracy_score(label_ids.detach().cpu().numpy(), preds.detach().cpu().numpy()) pbar.set_postfix(loss=loss.item(), acc=acc.item()) #nb_tr_examples += input_ids.size(0) nb_tr_examples += label_ids.size(0) nb_tr_steps += 1 if (step + 1) % args.gradient_accumulation_steps == 0: if args.fp16 or args.optimize_on_cpu: if args.fp16 and args.loss_scale != 1.0: # scale down gradients for fp16 training for param in model.parameters(): param.grad.data = param.grad.data / args.loss_scale is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True) if is_nan: logger.info("FP16 TRAINING: Nan in gradients, reducing loss scaling") args.loss_scale = args.loss_scale / 2 model.zero_grad() continue optimizer.step() copy_optimizer_params_to_model(model.named_parameters(), param_optimizer) else: optimizer.step() model.zero_grad() global_step += 1 torch.save(model.state_dict(), os.path.join(args.output_dir, "pytorch_model.bin")) if args.do_eval: from tqdm import tqdm, trange from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler from torch.utils.data.distributed import DistributedSampler eval_examples = processor.get_dev_examples(args.data_dir) eval_features = convert_examples_to_features( eval_examples, label_list, args.max_seq_length, tokenizer) # eval_features = convert_examples_to_siamese_features( # eval_examples, label_list, args.max_seq_length, tokenizer) logger.info("***** Running evaluation *****") logger.info(" Num examples = %d", len(eval_examples)) logger.info(" Batch size = %d", args.eval_batch_size) all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long) all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long) eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) # all_tokens_a = torch.tensor([f.tokens_a for f in eval_features], dtype=torch.long) # all_types_a = torch.tensor([f.types_a for f in eval_features], dtype=torch.long) # all_mask_a = torch.tensor([f.mask_a for f in eval_features], dtype=torch.long) # all_tokens_b = torch.tensor([f.tokens_b for f in eval_features], dtype=torch.long) # all_types_b = torch.tensor([f.types_b for f in eval_features], dtype=torch.long) # all_mask_b = torch.tensor([f.mask_b for f in eval_features], dtype=torch.long) # all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long) # eval_data = TensorDataset(all_tokens_a, all_types_a, all_mask_a, # all_tokens_b, all_types_b, all_mask_b, all_label_ids) if args.local_rank == -1: eval_sampler = SequentialSampler(eval_data) else: eval_sampler = DistributedSampler(eval_data) eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size) model.eval() eval_loss, eval_accuracy = 0, 0 nb_eval_steps, nb_eval_examples = 0, 0 y_true = [] y_pred = [] for batch in tqdm(eval_dataloader): # tokens_a, types_a, mask_a, tokens_b, types_b, mask_b, label_ids = batch input_ids, input_mask, segment_ids, label_ids = batch input_ids = input_ids.to(device) input_mask = input_mask.to(device) segment_ids = segment_ids.to(device) label_ids = label_ids.to(device) with torch.no_grad(): tmp_eval_loss, logits = model(input_ids, segment_ids, input_mask, label_ids) # tmp_eval_loss, logits = model(tokens_a, types_a, mask_a, # tokens_b, types_b, mask_b, label_ids) logits = logits.detach().cpu().numpy() label_ids = label_ids.to('cpu').numpy() preds = np.argmax(logits, axis=1) y_true.extend(label_ids) y_pred.extend(preds) tmp_eval_accuracy = accuracy(logits, label_ids) eval_loss += tmp_eval_loss.mean().item() eval_accuracy += tmp_eval_accuracy #nb_eval_examples += input_ids.size(0) nb_eval_examples += label_ids.size # np.array nb_eval_steps += 1 eval_loss = eval_loss / nb_eval_steps eval_accuracy = eval_accuracy / nb_eval_examples result = {'eval_loss': eval_loss, 'eval_accuracy': eval_accuracy, 'global_step': global_step, 'precision': precision_score(y_true, y_pred), 'recall': recall_score(y_true, y_pred), 'f1': f1_score(y_true, y_pred)} if tr_loss is not None: result['loss'] = tr_loss/nb_tr_steps output_eval_file = os.path.join(args.output_dir, "eval_results.txt") with open(output_eval_file, "a") as writer: logger.info("***** Eval results *****") writer.write("***** Eval results *****") for key in sorted(result.keys()): logger.info(" %s = %s", key, str(result[key])) writer.write("%s = %s\n" % (key, str(result[key]))) if args.do_submit: eval_examples = processor.get_test_examples(args.test_in_file) eval_features = convert_examples_to_features( eval_examples, label_list, args.max_seq_length, tokenizer) # eval_features = convert_examples_to_siamese_features( # eval_examples, label_list, args.max_seq_length, tokenizer) all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long) all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long) #eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) # all_tokens_a = torch.tensor([f.tokens_a for f in eval_features], dtype=torch.long) # all_types_a = torch.tensor([f.types_a for f in eval_features], dtype=torch.long) # all_mask_a = torch.tensor([f.mask_a for f in eval_features], dtype=torch.long) # all_tokens_b = torch.tensor([f.tokens_b for f in eval_features], dtype=torch.long) # all_types_b = torch.tensor([f.types_b for f in eval_features], dtype=torch.long) # all_mask_b = torch.tensor([f.mask_b for f in eval_features], dtype=torch.long) # all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long) # eval_data = TensorDataset(all_tokens_a, all_types_a, all_mask_a, # all_tokens_b, all_types_b, all_mask_b, all_label_ids) #eval_sampler = SequentialSampler(eval_data) #eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size) model.eval() y_pred = [] batch_size = args.eval_batch_size for i in range(0, len(all_label_ids), batch_size): input_ids = all_input_ids[i:i+batch_size] input_mask = all_input_mask[i:i+batch_size] segment_ids = all_segment_ids[i:i+batch_size] label_ids = all_label_ids[i:i+batch_size] # tokens_a = all_tokens_a[i:i+batch_size] # types_a = all_types_a[i:i+batch_size] # mask_a = all_mask_a[i:i+batch_size] # tokens_b = all_tokens_b[i:i+batch_size] # types_b = all_types_b[i:i+batch_size] # mask_b = all_mask_b[i:i+batch_size] with torch.no_grad(): logits = model(input_ids, segment_ids, input_mask) # logits = model(tokens_a, types_a, mask_a, # tokens_b, types_b, mask_b) logits = logits.detach().cpu().numpy() preds = np.argmax(logits, axis=1) y_pred.extend(preds) with open(args.test_out_file, "w") as writer: for i, y in enumerate(y_pred): writer.write('{}\t{}\n'.format(i+1, y))
def main(): parser = argparse.ArgumentParser() # Required parameters parser.add_argument("--train_file", default=None, type=str, required=True, help="The train file path") parser.add_argument("--eval_file", default=None, type=str, required=True, help="The dev file path") parser.add_argument("--eval_train_file", default=None, type=str, required=True, help="The train eval file path") parser.add_argument("--predict_file", default=None, type=str, required=False, help="The predict file path") parser.add_argument("--top_n", default=5, type=float, required=True, help="higher than threshold is classify 1,") parser.add_argument("--adam_epsilon", default=1e-8, type=float, required=False, help="adam eplisp") parser.add_argument( "--bert_config_file", default=None, type=str, required=True, help= "The config json file corresponding to the pre-trained BERT model. \n" "This specifies the model architecture.") parser.add_argument( "--bert_model", default=None, type=str, required=True, help= "The config json file corresponding to the pre-trained BERT model. \n" "This specifies the model architecture.") parser.add_argument( "--result_file", default=None, type=str, required=False, help="The result file that the BERT model was trained on.") parser.add_argument( "--vocab_file", default=None, type=str, required=True, help="The vocabulary file that the BERT model was trained on.") parser.add_argument( "--output_dir", default=None, type=str, required=True, help="The output directory where the model checkpoints will be written." ) # Other parameters parser.add_argument( "--init_checkpoint", default=None, type=str, help="Initial checkpoint (usually from a pre-trained BERT model).") parser.add_argument("--do_lower_case", default=False, action='store_true', help="Whether to lower case the input text.") parser.add_argument( "--max_seq_length", default=280, type=int, help="maximum total input sequence length after WordPiece tokenization." ) parser.add_argument("--do_train", default=False, action='store_true', help="Whether to run training.") parser.add_argument("--do_predict", default=False, action='store_true', help="Whether to run eval on the dev set.") parser.add_argument("--num_labels", default=1, type=int, help="mapping classify nums") parser.add_argument("--reduce_dim", default=64, type=int, help="from hidden size to this dimensions, reduce dim") parser.add_argument("--train_batch_size", default=32, type=int, help="Total batch size for training.") parser.add_argument("--eval_batch_size", default=8, type=int, help="Total batch size for eval.") parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument("--num_train_epochs", default=6.0, type=float, help="Total number of training epochs to perform.") parser.add_argument( "--warmup_proportion", default=0.1, type=float, help= "Proportion of training to perform linear learning rate warmup for. " "E.g., 0.1 = 10%% of training.") parser.add_argument("--weight_decay", default=0.01, type=float, help="Weight decay if we apply some.") parser.add_argument("--save_checkpoints_steps", default=1000, type=int, help="How often to save the model checkpoint.") parser.add_argument("--no_cuda", default=False, action='store_true', help="Whether not to use CUDA when available") parser.add_argument("--local_rank", type=int, default=-1, help="local_rank for distributed training on gpus") parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") parser.add_argument('--gradient_accumulation_steps', type=int, default=1, help="Number of updates steps to accumualte before") parser.add_argument( '--optimize_on_cpu', default=False, action='store_true', help="Whether to perform optimization and averages on CPU") parser.add_argument( '--fp16', default=False, action='store_true', help="Whether to use 16-bit float precision instead of 32-bit") parser.add_argument( '--loss_scale', type=float, default=128, help='Loss scale, positive power of 2 can improve fp16 convergence.') args = parser.parse_args() data_processor = DataProcessor(args.num_labels) if args.local_rank == -1 or args.no_cuda: device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") n_gpu = torch.cuda.device_count() else: device = torch.device("cuda", args.local_rank) n_gpu = 1 # Initializes the distributed backend which will take care of sychronizing nodes/GPUs torch.distributed.init_process_group(backend='nccl') if args.fp16: logger.info( "16-bits training currently not supported in distributed training" ) args.fp16 = False # (see https://github.com/pytorch/pytorch/pull/13496) logger.info("device %s n_gpu %d distributed training %r", device, n_gpu, bool(args.local_rank != -1)) if args.gradient_accumulation_steps < 1: raise ValueError( "Invalid gradient_accumulation_steps parameter: {}, should be >= 1" .format(args.gradient_accumulation_steps)) args.train_batch_size = int(args.train_batch_size / args.gradient_accumulation_steps) print(f'args.train_batch_size = {args.train_batch_size}') random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if n_gpu > 0: torch.cuda.manual_seed_all(args.seed) if not args.do_train and not args.do_predict: raise ValueError( "At least one of `do_train` or `do_eval` must be True.") bert_config = BertConfig.from_json_file(args.bert_config_file) # bert_config.reduce_dim = args.reduce_dim if args.max_seq_length > bert_config.max_position_embeddings: raise ValueError( "Cannot use sequence length {} because the BERT model was only trained up to sequence length {}" .format(args.max_seq_length, bert_config.max_position_embeddings)) if os.path.exists(args.output_dir) and os.listdir( args.output_dir) and args.do_train: raise ValueError( "Output directory ({}) already exists and is not empty.".format( args.output_dir)) if args.do_train: os.makedirs(args.output_dir, exist_ok=True) tokenizer = tokenization.FullTokenizer(vocab_file=args.vocab_file, do_lower_case=args.do_lower_case) def prepare_data(args, task_name='train'): if task_name == 'train': file_path = args.train_file elif task_name == 'eval': file_path = args.eval_file elif task_name == 'train_eval': file_path = args.eval_train_file elif task_name == 'predict': file_path = args.predict_file if os.path.isdir(file_path): examples = data_processor.read_file_dir(file_path, top_n=args.top_n) else: if task_name == 'predict': examples, example_map_ids = data_processor.read_predict_examples( file_path, top_n=args.top_n, task_name=task_name) else: examples, example_map_ids = data_processor.read_novel_examples( file_path, top_n=args.top_n, task_name=task_name) features = convert_examples_to_features(examples, args.max_seq_length, tokenizer) all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long) all_example_ids = torch.tensor([f.example_id for f in features], dtype=torch.long) if task_name in ['train', 'eval', 'train_eval', 'predict']: all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.long) datas = TensorDataset(all_example_ids, all_input_ids, all_input_mask, all_segment_ids, all_label_ids) else: datas = TensorDataset(all_example_ids, all_input_ids, all_input_mask, all_segment_ids) if task_name == 'train': if args.local_rank == -1: data_sampler = RandomSampler(datas) else: data_sampler = DistributedSampler(datas) dataloader = DataLoader(datas, sampler=data_sampler, batch_size=args.train_batch_size, drop_last=True) else: dataloader = DataLoader(datas, batch_size=args.eval_batch_size, drop_last=True) return (dataloader, example_map_ids) if task_name != 'train' else dataloader def accuracy(example_ids, logits, probs=None, data_type='eval'): logits = logits.tolist() example_ids = example_ids.tolist() assert len(logits) == len(example_ids) classify_name = ['no_answer', 'yes_answer'] labels, text_a, novel_names = [], [], [] map_dicts = example_map_ids if data_type == 'eval' else train_example_map_ids for i in example_ids: example = map_dicts[i] labels.append(example.label) text_a.append("||".join(example.text_a)) novel_names.append(example.name) write_data = pd.DataFrame({ "text_a": text_a, "labels": labels, "logits": logits, "novel_names": novel_names, }) write_data['yes_or_no'] = write_data['labels'] == write_data['logits'] if probs is not None: write_data['logits'] = probs.tolist() write_data.to_csv(args.result_file, index=False) assert len(labels) == len(logits) result = classification_report(labels, logits, target_names=classify_name) return result def eval_model(model, eval_dataloader, device, data_type='eval'): model.eval() eval_loss = 0 all_logits = [] all_example_ids = [] all_probs = [] accuracy_result = None batch_count = 0 for step, batch in enumerate(tqdm(eval_dataloader, desc="evaluating")): example_ids, input_ids, input_mask, segment_ids, label_ids = batch label_ids = label_ids.to(device) input_ids = input_ids.to(device) input_mask = input_mask.to(device) segment_ids = segment_ids.to(device) if not args.do_train: label_ids = None with torch.no_grad(): tmp_eval_loss, logits = model(input_ids, segment_ids, input_mask, labels=label_ids) argmax_logits = torch.argmax(logits, dim=1) first_indices = torch.arange(argmax_logits.size()[0]) logits_probs = logits[first_indices, argmax_logits] if args.do_train: eval_loss += tmp_eval_loss.mean().item() all_logits.append(argmax_logits) all_example_ids.append(example_ids) else: all_logits.append(argmax_logits) all_example_ids.append(example_ids) all_probs.append(logits_probs) batch_count += 1 if all_logits: all_logits = torch.cat(all_logits, dim=0) all_example_ids = torch.cat(all_example_ids, dim=0) all_probs = torch.cat(all_probs, dim=0) if len(all_probs) else None accuracy_result = accuracy(all_example_ids, all_logits, probs=all_probs, data_type=data_type) eval_loss /= batch_count print(f'\n========= {data_type} acc ============') print(f'{accuracy_result}') return eval_loss, accuracy_result, all_logits def predict_model(model, predict_dataloader, device): model.eval() all_label_ids = [] all_probs = [] scene_cut_indexs = [] is_cut_probs = [] for step, batch in enumerate( tqdm(predict_dataloader, desc="predicting")): example_ids, input_ids, input_mask, segment_ids, label_ids = batch # print(f'input_ids = {input_ids.shape}') # print(f'input_mask = {input_mask.shape}') # print(f'segment_ids = {segment_ids.shape}') # print(f'label_ids = {label_ids.shape}') with torch.no_grad(): _, logits = model(input_ids, segment_ids, input_mask, labels=None) argmax_logits = torch.argmax(logits, dim=1) first_indices = torch.arange(argmax_logits.size()[0]) logits_probs = logits[first_indices, argmax_logits] all_label_ids.extend(label_ids.tolist()) all_probs.extend(logits_probs.tolist()) for label, prob, pred_label in zip(label_ids.tolist(), logits_probs.tolist(), argmax_logits.tolist()): if pred_label: scene_cut_indexs.append(label) is_cut_probs.append(prob) # print(f'prob = {prob}') # print(f'pred label = {pred_label}') # print(f'sent = {predict_example_map_ids[label]}') print(f'all_example_ids = {len(all_label_ids)}') print(f'all_probs = {len(all_probs)}') assert len(all_label_ids) == len(all_probs) return scene_cut_indexs, is_cut_probs if args.do_train: train_dataloader = prepare_data(args, task_name='train') num_train_steps = int( len(train_dataloader) / args.gradient_accumulation_steps * args.num_train_epochs) model = BertForSequenceClassification(bert_config, num_labels=data_processor.num_labels) new_state_dict = model.state_dict() init_state_dict = torch.load( os.path.join(args.bert_model, 'pytorch_model.bin')) for k, v in init_state_dict.items(): if k in new_state_dict: print(f'k in = {k} v in shape = {v.shape}') new_state_dict[k] = v model.load_state_dict(new_state_dict) init_model_token_type(model, type_vocab_size=args.top_n) for k, v in model.state_dict().items(): print(f'k = {k}, v shape {v.shape}') if args.fp16: model.half() if args.do_predict: model_path = os.path.join(args.output_dir, WEIGHTS_NAME) new_state_dict = torch.load(model_path) new_state_dict = dict([(k[7:], v) if k.startswith('module') else (k, v) for k, v in new_state_dict.items()]) model.load_state_dict(new_state_dict) model.to(device) if args.local_rank != -1: model = torch.nn.parallel.DistributedDataParallel( model, device_ids=[args.local_rank], output_device=args.local_rank) elif n_gpu > 1 and not device == 'cpu': model = torch.nn.DataParallel(model) # # model = BalancedDataParallel(1, model, dim=0).to(device) eval_dataloader, example_map_ids = prepare_data(args, task_name='eval') train_eval_dataloader, train_example_map_ids = prepare_data( args, task_name='train_eval') def get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, last_epoch=-1): """ Create a schedule with a learning rate that decreases linearly after linearly increasing during a warmup period. """ def lr_lambda(current_step): if current_step < num_warmup_steps: return float(current_step) / float(max(1, num_warmup_steps)) return max( 0.0, float(num_training_steps - current_step) / float(max(1, num_training_steps - num_warmup_steps))) return LambdaLR(optimizer, lr_lambda, last_epoch) def get_optimizers(num_training_steps: int, model): no_decay = ["bias", "LayerNorm.weight"] optimizer_grouped_parameters = [ { "params": [ p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay) ], "weight_decay": args.weight_decay, }, { "params": [ p for n, p in model.named_parameters() if any(nd in n for nd in no_decay) ], "weight_decay": 0.0, }, ] optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) warmup_steps = args.warmup_proportion * num_training_steps scheduler = get_linear_schedule_with_warmup( optimizer, num_warmup_steps=warmup_steps, num_training_steps=num_training_steps) return optimizer, scheduler if args.do_train: optimizer, scheduler = get_optimizers( num_training_steps=num_train_steps, model=model) output_config_file = os.path.join(args.output_dir, CONFIG_NAME) # eval_loss, acc, _ = eval_model(model, eval_dataloader, device) # logger.info(f'初始开发集loss: {eval_loss}') for epoch in trange(int(args.num_train_epochs), desc="Epoch"): model.train() torch.cuda.empty_cache() model_save_path = os.path.join(args.output_dir, f"{WEIGHTS_NAME}.{epoch}") tr_loss = 0 train_batch_count = 0 for step, batch in enumerate( tqdm(train_dataloader, desc="training")): _, input_ids, input_mask, segment_ids, label_ids = batch label_ids = label_ids.to(device) input_ids = input_ids.to(device) input_mask = input_mask.to(device) segment_ids = segment_ids.to(device) loss, _ = model(input_ids, segment_ids, input_mask, labels=label_ids) if n_gpu > 1: loss = loss.mean() if args.fp16 and args.loss_scale != 1.0: loss = loss * args.loss_scale if args.gradient_accumulation_steps > 1: loss = loss / args.gradient_accumulation_steps loss.backward() tr_loss += loss.item() if (step + 1) % args.gradient_accumulation_steps == 0: optimizer.step() scheduler.step() model.zero_grad() train_batch_count += 1 tr_loss /= train_batch_count eval_loss, acc, _ = eval_model(model, eval_dataloader, device) eval_model(model, train_eval_dataloader, device, data_type='train_eval') logger.info( f'训练loss: {tr_loss}, 开发集loss:{eval_loss} 训练轮数:{epoch + 1}/{int(args.num_train_epochs)}' ) model_to_save = model.module if hasattr(model, 'module') else model torch.save(model.state_dict(), model_save_path) if epoch == 0: model_to_save.config.to_json_file(output_config_file) tokenizer.save_vocabulary(args.output_dir) if args.do_predict: # eval_model(model, train_eval_dataloader, device, data_type='train_eval') # for root, dir_, files in os.walk(''): # for file in files: # arg.predict_file = os.path.join(root, file) predict_dataloader, predict_example_map_ids = prepare_data( args, task_name='predict') scene_cut_indexs, is_cut_probs = predict_model(model, predict_dataloader, device) assert len(scene_cut_indexs) == len(is_cut_probs) predict_novel_name = os.path.join( os.path.split(args.predict_file)[0], 'scene_cut_datas', os.path.split(args.predict_file)[-1]) for i, index in enumerate(scene_cut_indexs): predict_example_map_ids[ index] = f'######## {round(is_cut_probs[i], 3)} {predict_example_map_ids[index]}' with open(predict_novel_name, 'w', encoding='utf-8') as f: for i, line in enumerate(predict_example_map_ids): f.write(line)
def main(): parser = argparse.ArgumentParser() # Required parameters parser.add_argument( "--data_dir", default=None, type=str, required=True, help= "The input data dir. Should contain the .tsv files (or other data files) for the task." ) parser.add_argument( "--bert_config_file", default=None, type=str, required=True, help= "The config json file corresponding to the pre-trained BERT model. \n" "This specifies the model architecture.") parser.add_argument("--task_name", default=None, type=str, required=True, help="The name of the task to train.") parser.add_argument( "--vocab_file", default=None, type=str, required=True, help="The vocabulary file that the BERT model was trained on.") parser.add_argument( "--output_dir", default=None, type=str, required=True, help="The output directory where the model checkpoints will be written." ) # Other parameters parser.add_argument( "--init_checkpoint", default=None, type=str, help="Initial checkpoint (usually from a pre-trained BERT model).") parser.add_argument( "--do_lower_case", default=False, action='store_true', help= "Whether to lower case the input text. True for uncased models, False for cased models." ) parser.add_argument( "--max_seq_length", default=128, type=int, help= "The maximum total input sequence length after WordPiece tokenization. \n" "Sequences longer than this will be truncated, and sequences shorter \n" "than this will be padded.") parser.add_argument("--do_train", default=False, action='store_true', help="Whether to run training.") parser.add_argument("--do_eval", default=False, action='store_true', help="Whether to run eval on the dev set.") parser.add_argument("--train_batch_size", default=32, type=int, help="Total batch size for training.") parser.add_argument("--eval_batch_size", default=8, type=int, help="Total batch size for eval.") parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument("--num_train_epochs", default=3.0, type=float, help="Total number of training epochs to perform.") parser.add_argument( "--warmup_proportion", default=0.1, type=float, help= "Proportion of training to perform linear learning rate warmup for. " "E.g., 0.1 = 10%% of training.") parser.add_argument("--save_checkpoints_steps", default=1000, type=int, help="How often to save the model checkpoint.") parser.add_argument("--no_cuda", default=False, action='store_true', help="Whether not to use CUDA when available") parser.add_argument("--local_rank", type=int, default=-1, help="local_rank for distributed training on gpus") parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") parser.add_argument( '--gradient_accumulation_steps', type=int, default=1, help= "Number of updates steps to accumualte before performing a backward/update pass." ) parser.add_argument( '--optimize_on_cpu', default=False, action='store_true', help= "Whether to perform optimization and keep the optimizer averages on CPU" ) parser.add_argument( '--fp16', default=False, action='store_true', help="Whether to use 16-bit float precision instead of 32-bit") parser.add_argument( '--loss_scale', type=float, default=128, help= 'Loss scaling, positive power of 2 values can improve fp16 convergence.' ) args = parser.parse_args() processors = { "cola": ColaProcessor, "mnli": MnliProcessor, "mrpc": MrpcProcessor, "news": NewsProcessor, "car": CarProcessor, } if args.local_rank == -1 or args.no_cuda: device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") n_gpu = torch.cuda.device_count() else: device = torch.device("cuda", args.local_rank) n_gpu = 1 # Initializes the distributed backend which will take care of sychronizing nodes/GPUs # torch.distributed.init_process_group(backend='nccl') if args.fp16: logger.info( "16-bits training currently not supported in distributed training" ) args.fp16 = False # (see https://github.com/pytorch/pytorch/pull/13496) logger.info("device %s n_gpu %d distributed training %r", device, n_gpu, bool(args.local_rank != -1)) if args.gradient_accumulation_steps < 1: raise ValueError( "Invalid gradient_accumulation_steps parameter: {}, should be >= 1" .format(args.gradient_accumulation_steps)) args.train_batch_size = int(args.train_batch_size / args.gradient_accumulation_steps) random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if n_gpu > 0: torch.cuda.manual_seed_all(args.seed) if not args.do_train and not args.do_eval: raise ValueError( "At least one of `do_train` or `do_eval` must be True.") bert_config = BertConfig.from_json_file(args.bert_config_file) if args.max_seq_length > bert_config.max_position_embeddings: raise ValueError( "Cannot use sequence length {} because the BERT model was only trained up to sequence length {}" .format(args.max_seq_length, bert_config.max_position_embeddings)) if os.path.exists(args.output_dir) and os.listdir(args.output_dir): raise ValueError( "Output directory ({}) already exists and is not empty.".format( args.output_dir)) os.makedirs(args.output_dir, exist_ok=True) task_name = args.task_name.lower() if task_name not in processors: raise ValueError("Task not found: %s" % (task_name)) processor = processors[task_name]() tokenizer = tokenization.FullTokenizer(vocab_file=args.vocab_file, do_lower_case=args.do_lower_case) train_examples = None num_train_steps = None if args.do_train: train_examples = processor.get_train_examples(args.data_dir) num_train_steps = int( len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps * args.num_train_epochs) label_list = processor.get_labels() print("label_list.size:%d\n" % (len(label_list))) # Prepare model model = BertForSequenceClassification(bert_config, len(label_list)) if args.init_checkpoint is not None: model.bert.load_state_dict( torch.load(args.init_checkpoint, map_location='cpu')) if args.fp16: model.half() model.to(device) # if args.local_rank != -1: # model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], # output_device=args.local_rank) # elif n_gpu > 1: # model = torch.nn.DataParallel(model) # Prepare optimizer if args.fp16: param_optimizer = [ (n, param.clone().detach().to('cpu').float().requires_grad_()) for n, param in model.named_parameters() ] elif args.optimize_on_cpu: param_optimizer = [(n, param.clone().detach().to('cpu').requires_grad_()) for n, param in model.named_parameters()] else: param_optimizer = list(model.named_parameters()) no_decay = ['bias', 'gamma', 'beta'] optimizer_grouped_parameters = [{ 'params': [p for n, p in param_optimizer if n not in no_decay], 'weight_decay_rate': 0.01 }, { 'params': [p for n, p in param_optimizer if n in no_decay], 'weight_decay_rate': 0.0 }] optimizer = BERTAdam(optimizer_grouped_parameters, lr=args.learning_rate, warmup=args.warmup_proportion, t_total=num_train_steps) global_step = 0 if args.do_train: train_features = convert_examples_to_features(train_examples, label_list, args.max_seq_length, tokenizer) logger.info("***** Running training *****") logger.info(" Num examples = %d", len(train_examples)) logger.info(" Batch size = %d", args.train_batch_size) logger.info(" Num steps = %d", num_train_steps) all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long) all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long) train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) if args.local_rank == -1: train_sampler = RandomSampler(train_data) else: train_sampler = RandomSampler(train_data) # train_sampler = DistributedSampler(train_data) train_data_loader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size) model.train() for _ in trange(int(args.num_train_epochs), desc="Epoch"): tr_loss = 0 nb_tr_examples, nb_tr_steps = 0, 0 for step, batch in enumerate( tqdm(train_data_loader, desc="Iteration")): batch = tuple(t.to(device) for t in batch) input_ids, input_mask, segment_ids, label_ids = batch loss, _ = model(input_ids, segment_ids, input_mask, label_ids) if n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu. if args.fp16 and args.loss_scale != 1.0: # rescale loss for fp16 training # see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html loss = loss * args.loss_scale if args.gradient_accumulation_steps > 1: loss = loss / args.gradient_accumulation_steps loss.backward() tr_loss += loss.item() nb_tr_examples += input_ids.size(0) nb_tr_steps += 1 if (step + 1) % args.gradient_accumulation_steps == 0: if args.fp16 or args.optimize_on_cpu: if args.fp16 and args.loss_scale != 1.0: # scale down gradients for fp16 training for param in model.parameters(): param.grad.data = param.grad.data / args.loss_scale is_nan = set_optimizer_params_grad( param_optimizer, model.named_parameters(), test_nan=True) if is_nan: logger.info( "FP16 TRAINING: Nan in gradients, reducing loss scaling" ) args.loss_scale = args.loss_scale / 2 model.zero_grad() continue optimizer.step() copy_optimizer_params_to_model( model.named_parameters(), param_optimizer) else: optimizer.step() model.zero_grad() global_step += 1 torch.save(model.state_dict(), os.path.join(args.output_dir, "model.pkl")) if args.do_eval: eval_examples = processor.get_dev_examples(args.data_dir) eval_features = convert_examples_to_features(eval_examples, label_list, args.max_seq_length, tokenizer) logger.info("***** Running evaluation *****") logger.info(" Num examples = %d", len(eval_examples)) logger.info(" Batch size = %d", args.eval_batch_size) all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long) all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long) eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) if args.local_rank == -1: eval_sampler = SequentialSampler(eval_data) else: eval_sampler = SequentialSampler(eval_data) # eval_sampler = DistributedSampler(eval_data) eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size) model.eval() eval_loss, eval_accuracy = 0, 0 nb_eval_steps, nb_eval_examples = 0, 0 for input_ids, input_mask, segment_ids, label_ids in eval_dataloader: input_ids = input_ids.to(device) input_mask = input_mask.to(device) segment_ids = segment_ids.to(device) label_ids = label_ids.to(device) with torch.no_grad(): tmp_eval_loss, logits = model(input_ids, segment_ids, input_mask, label_ids) logits = logits.detach().cpu().numpy() label_ids = label_ids.to('cpu').numpy() tmp_eval_accuracy = accuracy(logits, label_ids) eval_loss += tmp_eval_loss.mean().item() eval_accuracy += tmp_eval_accuracy nb_eval_examples += input_ids.size(0) nb_eval_steps += 1 eval_loss = eval_loss / nb_eval_steps eval_accuracy = eval_accuracy / nb_eval_examples result = { 'eval_loss': eval_loss, 'eval_accuracy': eval_accuracy, 'global_step': global_step, 'loss': tr_loss / nb_tr_steps } output_eval_file = os.path.join(args.output_dir, "eval_results.txt") with open(output_eval_file, "w") as writer: logger.info("***** Eval results *****") for key in sorted(result.keys()): logger.info(" %s = %s", key, str(result[key])) writer.write("%s = %s\n" % (key, str(result[key])))
class Instructor: def __init__(self, args): self.opt = args self.writer = SummaryWriter(log_dir=self.opt.output_dir) # tensorboard bert_config = BertConfig.from_json_file(args.bert_config_file) if args.max_seq_length > bert_config.max_position_embeddings: raise ValueError( "Cannot use sequence length {} because the BERT model was only trained up to sequence length {}" .format(args.max_seq_length, bert_config.max_position_embeddings)) # if os.path.exists(args.output_dir) and os.listdir(args.output_dir): # raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir)) # os.makedirs(args.output_dir, exist_ok=True) self.dataset = ReadData(self.opt) # Read the data and preprocess it self.num_train_steps = None self.num_train_steps = int( len(self.dataset.train_examples) / self.opt.train_batch_size / self.opt.gradient_accumulation_steps * self.opt.num_train_epochs) self.opt.label_size = len(self.dataset.label_list) args.output_dim = len(self.dataset.label_list) print("label size: {}".format(args.output_dim)) # 初始化模型 print("initialize model ...") if args.model_class == BertForSequenceClassification: self.model = BertForSequenceClassification( bert_config, len(self.dataset.label_list)) else: self.model = model_classes[args.model_name](bert_config, args) if args.init_checkpoint is not None: self.model.bert.load_state_dict( torch.load(args.init_checkpoint, map_location='cpu')) if args.fp16: self.model.half() # 冻结参数 # for name, p in self.model.named_parameters(): # if name.startswith('bert.encoder.layer.11') or name.startswith('bert.encoder.layer.10') or name.startswith('bert.encoder.layer.9') or name.startswith('bert.encoder.layer.8'): # 冻结最后一层 # p.requires_grad = False # 计算模型的参数个数 n_trainable_params, n_nontrainable_params = 0, 0 for p in self.model.parameters(): n_params = torch.prod(torch.tensor( p.shape)) # torch.prod()表示计算所有元素的乘积 if p.requires_grad: # 是否需要求梯度 n_trainable_params += n_params else: n_nontrainable_params += n_params print('n_trainable_params: {0}, n_nontrainable_params: {1}'.format( n_trainable_params, n_nontrainable_params)) self.model.to(args.device) # 并行化 if self.opt.n_gpu > 1: self.model = torch.nn.DataParallel(self.model, device_ids=self.opt.gpu_id) # Prepare optimizer if args.fp16: self.param_optimizer = [(n, param.clone().detach().to('cpu').float().requires_grad_()) \ for n, param in self.model.named_parameters()] elif args.optimize_on_cpu: self.param_optimizer = [(n, param.clone().detach().to('cpu').requires_grad_()) \ for n, param in self.model.named_parameters()] else: self.param_optimizer = list(self.model.named_parameters()) no_decay = ['bias', 'gamma', 'beta'] optimizer_grouped_parameters = [{ 'params': [p for n, p in self.param_optimizer if n not in no_decay], 'weight_decay_rate': 0.01 }, { 'params': [p for n, p in self.param_optimizer if n in no_decay], 'weight_decay_rate': 0.0 }] self.optimizer = BERTAdam(optimizer_grouped_parameters, lr=args.learning_rate, warmup=args.warmup_proportion, t_total=self.num_train_steps) # 配置自己模型的优化器 # [p for pname, p in self.param_optimizer if not pname.startswith('module.bert')] # self.optimizer_me = torch.optim.Adam( # [{'params': [p for pname, p in self.param_optimizer if not pname.startswith('module.bert')]}], lr=0.001, # weight_decay=0) # self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(self.optimizer_me, mode='max', # patience=3) # 3个epoch后,所监测的值停止增加时自动调整学习率 self.global_step = 0 # 初始化全局步数为 0 self.max_test_acc = 0 self.max_test_f1 = 0 def do_train(self): # 训练模型 # for _ in trange(int(args.num_train_epochs), desc="Epoch"): for i_epoch in range(int(args.num_train_epochs)): print('>' * 100) print('epoch: ', i_epoch) tr_loss = 0 train_accuracy = 0 nb_tr_examples, nb_tr_steps = 0, 0 y_pred = [] y_true = [] self.model.train() # 让模型处于训练状态,因为每跑完一个epoch就会处于测试状态 for step, batch in enumerate( tqdm(self.dataset.train_dataloader, desc="Training")): # batch = tuple(t.to(self.opt.device) for t in batch) input_ids, input_mask, segment_ids, label_ids, \ input_t_ids, input_t_mask, segment_t_ids, \ input_without_t_ids, input_without_t_mask, segment_without_t_ids, \ input_left_t_ids, input_left_t_mask, segment_left_t_ids, \ input_right_t_ids, input_right_t_mask, segment_right_t_ids, \ input_left_ids, input_left_mask, segment_left_ids = batch input_ids = input_ids.to(self.opt.device) segment_ids = segment_ids.to(self.opt.device) input_mask = input_mask.to(self.opt.device) label_ids = label_ids.to(self.opt.device) if self.opt.model_class in [ BertForSequenceClassification, CNN ]: loss, logits = self.model(input_ids, segment_ids, input_mask, label_ids) else: input_t_ids = input_t_ids.to(self.opt.device) input_t_mask = input_t_mask.to(self.opt.device) segment_t_ids = segment_t_ids.to(self.opt.device) if self.opt.model_class == MemNet: input_without_t_ids = input_without_t_ids.to( self.opt.device) input_without_t_mask = input_without_t_mask.to( self.opt.device) segment_without_t_ids = segment_without_t_ids.to( self.opt.device) loss, logits = self.model(input_without_t_ids, segment_without_t_ids, input_without_t_mask, label_ids, input_t_ids, input_t_mask, segment_t_ids) elif self.opt.model_class in [Cabasc]: input_left_t_ids = input_left_t_ids.to(self.opt.device) input_left_t_mask = input_left_t_mask.to( self.opt.device) segment_left_t_ids = segment_left_t_ids.to( self.opt.device) input_right_t_ids = input_right_t_ids.to( self.opt.device) input_right_t_mask = input_right_t_mask.to( self.opt.device) segment_right_t_ids = segment_right_t_ids.to( self.opt.device) loss, logits = self.model( input_ids, segment_ids, input_mask, label_ids, input_t_ids, input_t_mask, segment_t_ids, input_left_t_ids, input_left_t_mask, segment_left_t_ids, input_right_t_ids, input_right_t_mask, segment_right_t_ids) elif self.opt.model_class in [ RAM, TNet_LF, MGAN, TT, MLP, TD_BERT, TD_BERT_QA, DTD_BERT ]: input_left_ids = input_left_ids.to(self.opt.device) input_left_mask = input_left_mask.to(self.opt.device) segment_left_ids = segment_left_ids.to(self.opt.device) loss, logits = self.model( input_ids, segment_ids, input_mask, label_ids, input_t_ids, input_t_mask, segment_t_ids, input_left_ids, input_left_mask, segment_left_ids) else: loss, logits = self.model(input_ids, segment_ids, input_mask, label_ids, input_t_ids, input_t_mask, segment_t_ids) if self.opt.n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu. if args.fp16 and args.loss_scale != 1.0: # rescale loss for fp16 training # see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html loss = loss * args.loss_scale if args.gradient_accumulation_steps > 1: loss = loss / args.gradient_accumulation_steps tr_loss += loss.item() loss.backward() nb_tr_examples += input_ids.size(0) nb_tr_steps += 1 # 计算准确率 logits = logits.detach().cpu().numpy() label_ids = label_ids.to('cpu').numpy() tmp_train_accuracy = accuracy(logits, label_ids) y_pred.extend(np.argmax(logits, axis=1)) y_true.extend(label_ids) train_accuracy += tmp_train_accuracy if (step + 1) % args.gradient_accumulation_steps == 0: if args.fp16 or args.optimize_on_cpu: if args.fp16 and args.loss_scale != 1.0: # scale down gradients for fp16 training for param in self.model.parameters(): param.grad.data = param.grad.data / args.loss_scale is_nan = set_optimizer_params_grad( self.param_optimizer, self.model.named_parameters(), test_nan=True) if is_nan: logger.info( "FP16 TRAINING: Nan in gradients, reducing loss scaling" ) args.loss_scale = args.loss_scale / 2 self.model.zero_grad() continue self.optimizer.step() # self.optimizer_me.step() copy_optimizer_params_to_model( self.model.named_parameters(), self.param_optimizer) else: self.optimizer.step() # self.optimizer_me.step() self.model.zero_grad() self.global_step += 1 train_accuracy = train_accuracy / nb_tr_examples train_f1 = f1_score(y_true, y_pred, average='macro', labels=np.unique(y_true)) result = self.do_eval() # 每跑完一轮,测试一次 tr_loss = tr_loss / nb_tr_steps # self.scheduler.step(result['eval_accuracy']) # 监测验证集的精度 self.writer.add_scalar('train_loss', tr_loss, i_epoch) self.writer.add_scalar('train_accuracy', train_accuracy, i_epoch) self.writer.add_scalar('eval_accuracy', result['eval_accuracy'], i_epoch) self.writer.add_scalar('eval_loss', result['eval_loss'], i_epoch) # self.writer.add_scalar('lr', self.optimizer_me.param_groups[0]['lr'], i_epoch) print( "Results: train_acc: {0:.6f} | train_f1: {1:.6f} | train_loss: {2:.6f} | eval_accuracy: {3:.6f} | eval_loss: {4:.6f} | eval_f1: {5:.6f} | max_test_acc: {6:.6f} | max_test_f1: {7:.6f}" .format(train_accuracy, train_f1, tr_loss, result['eval_accuracy'], result['eval_loss'], result['eval_f1'], self.max_test_acc, self.max_test_f1)) def do_eval(self): # 测试准确率 self.model.eval() eval_loss, eval_accuracy = 0, 0 nb_eval_steps, nb_eval_examples = 0, 0 # confidence = [] y_pred = [] y_true = [] for batch in tqdm(self.dataset.eval_dataloader, desc="Evaluating"): # batch = tuple(t.to(self.opt.device) for t in batch) input_ids, input_mask, segment_ids, label_ids, \ input_t_ids, input_t_mask, segment_t_ids, \ input_without_t_ids, input_without_t_mask, segment_without_t_ids, \ input_left_t_ids, input_left_t_mask, segment_left_t_ids, \ input_right_t_ids, input_right_t_mask, segment_right_t_ids, \ input_left_ids, input_left_mask, segment_left_ids = batch input_ids = input_ids.to(self.opt.device) segment_ids = segment_ids.to(self.opt.device) input_mask = input_mask.to(self.opt.device) label_ids = label_ids.to(self.opt.device) with torch.no_grad(): # 不计算梯度 if self.opt.model_class in [ BertForSequenceClassification, CNN ]: loss, logits = self.model(input_ids, segment_ids, input_mask, label_ids) else: input_t_ids = input_t_ids.to(self.opt.device) input_t_mask = input_t_mask.to(self.opt.device) segment_t_ids = segment_t_ids.to(self.opt.device) if self.opt.model_class == MemNet: input_without_t_ids = input_without_t_ids.to( self.opt.device) input_without_t_mask = input_without_t_mask.to( self.opt.device) segment_without_t_ids = segment_without_t_ids.to( self.opt.device) loss, logits = self.model(input_without_t_ids, segment_without_t_ids, input_without_t_mask, label_ids, input_t_ids, input_t_mask, segment_t_ids) elif self.opt.model_class in [Cabasc]: input_left_t_ids = input_left_t_ids.to(self.opt.device) input_left_t_ids = input_left_t_ids.to(self.opt.device) input_left_t_mask = input_left_t_mask.to( self.opt.device) segment_left_t_ids = segment_left_t_ids.to( self.opt.device) input_right_t_ids = input_right_t_ids.to( self.opt.device) input_right_t_mask = input_right_t_mask.to( self.opt.device) segment_right_t_ids = segment_right_t_ids.to( self.opt.device) loss, logits = self.model( input_ids, segment_ids, input_mask, label_ids, input_t_ids, input_t_mask, segment_t_ids, input_left_t_ids, input_left_t_mask, segment_left_t_ids, input_right_t_ids, input_right_t_mask, segment_right_t_ids) elif self.opt.model_class in [ RAM, TNet_LF, MGAN, TT, MLP, TD_BERT, TD_BERT_QA, DTD_BERT ]: input_left_ids = input_left_ids.to(self.opt.device) input_left_mask = input_left_mask.to(self.opt.device) segment_left_ids = segment_left_ids.to(self.opt.device) loss, logits = self.model( input_ids, segment_ids, input_mask, label_ids, input_t_ids, input_t_mask, segment_t_ids, input_left_ids, input_left_mask, segment_left_ids) else: loss, logits = self.model(input_ids, segment_ids, input_mask, label_ids, input_t_ids, input_t_mask, segment_t_ids) # with torch.no_grad(): # 不计算梯度 # if self.opt.model_class in [BertForSequenceClassification, CNN]: # loss, logits = self.model(input_ids, segment_ids, input_mask, label_ids) # else: # loss, logits = self.model(input_ids, segment_ids, input_mask, labels=label_ids, # input_t_ids=input_t_ids, # input_t_mask=input_t_mask, segment_t_ids=segment_t_ids) # confidence.extend(torch.nn.Softmax(dim=1)(logits)[:, 1].tolist()) # 获取 positive 类的置信度 # loss = F.cross_entropy(logits, label_ids, size_average=False) # 计算mini-batch的loss总和 if self.opt.n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu. if args.fp16 and args.loss_scale != 1.0: # rescale loss for fp16 training # see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html loss = loss * args.loss_scale if args.gradient_accumulation_steps > 1: loss = loss / args.gradient_accumulation_steps logits = logits.detach().cpu().numpy() label_ids = label_ids.to('cpu').numpy() tmp_eval_accuracy = accuracy(logits, label_ids) y_pred.extend(np.argmax(logits, axis=1)) y_true.extend(label_ids) # eval_loss += tmp_eval_loss.mean().item() eval_loss += loss.item() eval_accuracy += tmp_eval_accuracy nb_eval_examples += input_ids.size(0) nb_eval_steps += 1 # eval_loss = eval_loss / len(self.dataset.eval_examples) test_f1 = f1_score(y_true, y_pred, average='macro', labels=np.unique(y_true)) eval_loss = eval_loss / nb_eval_steps eval_accuracy = eval_accuracy / nb_eval_examples if eval_accuracy > self.max_test_acc: self.max_test_acc = eval_accuracy if self.opt.do_predict: # 测试模式才保存模型 torch.save(self.model, self.opt.model_save_path) if test_f1 > self.max_test_f1: self.max_test_f1 = test_f1 result = { 'eval_loss': eval_loss, 'eval_accuracy': eval_accuracy, 'eval_f1': test_f1, } # output_eval_file = os.path.join(args.output_dir, "eval_results.txt") # with open(output_eval_file, "w") as writer: # logger.info("***** Eval results *****") # for key in sorted(result.keys()): # logger.info(" %s = %s", key, str(result[key])) # writer.write("%s = %s\n" % (key, str(result[key]))) # print("Eval results ==> eval_accuracy: {0}, eval_loss: {1}, max_test_acc: {2}".format( # result['eval_accuracy'], result['eval_loss'], self.max_test_acc)) return result def do_predict(self): # 加载保存的模型进行预测,获得准确率 # 读测试集的数据 # dataset = ReadData(self.opt) # 这个方法有点冗余了,读取了所有的数据,包括训练集 # Load model saved_model = torch.load(self.opt.model_save_path) saved_model.to(self.opt.device) saved_model.eval() nb_test_examples = 0 test_accuracy = 0 for batch in tqdm(self.dataset.eval_dataloader, desc="Testing"): batch = tuple(t.to(self.opt.device) for t in batch) input_ids, input_mask, segment_ids, label_ids, input_t_ids, input_t_mask, segment_t_ids = batch with torch.no_grad(): # Do not calculate gradient if self.opt.model_class in [ BertForSequenceClassification, CNN ]: _, logits = saved_model(input_ids, segment_ids, input_mask, label_ids) else: _, logits = saved_model(input_ids, segment_ids, input_mask, labels=label_ids, input_t_ids=input_t_ids, input_t_mask=input_t_mask, segment_t_ids=segment_t_ids) logits = logits.detach().cpu().numpy() label_ids = label_ids.to('cpu').numpy() tmp_test_accuracy = accuracy(logits, label_ids) test_accuracy += tmp_test_accuracy nb_test_examples += input_ids.size(0) test_accuracy = test_accuracy / nb_test_examples return test_accuracy def run(self): print('> training arguments:') for arg in vars(self.opt): print('>>> {0}: {1}'.format(arg, getattr(self.opt, arg))) self.do_train() print('>' * 100) if self.opt.do_predict: test_accuracy = self.do_predict() print("Test Set Accuracy: {}".format(test_accuracy)) print("Max validate Set Acc: {0}".format( self.max_test_acc)) # Output the final test accuracy self.writer.close() # return self.max_test_acc return self.max_test_f1
def main(): parser = argparse.ArgumentParser() ## Required parameters parser.add_argument("--data_dir", default=None, type=str, required=True, help="The input data dir. Should contain the .tsv files (or other data files) for the task.") parser.add_argument("--bert_config_file", default=None, type=str, required=True, help="The config json file corresponding to the pre-trained BERT model. \n" "This specifies the model architecture.") parser.add_argument("--task_name", default=None, type=str, required=True, help="The name of the task to train.") parser.add_argument("--vocab_file", default=None, type=str, required=True, help="The vocabulary file that the BERT model was trained on.") parser.add_argument("--output_dir", default=None, type=str, required=True, help="The output directory where the model checkpoints will be written.") ## Other parameters parser.add_argument("--init_checkpoint", default=None, type=str, help="Initial checkpoint (usually from a pre-trained BERT model).") parser.add_argument("--do_lower_case", default=False, action='store_true', help="Whether to lower case the input text. True for uncased models, False for cased models.") parser.add_argument("--max_seq_length", default=128, type=int, help="The maximum total input sequence length after WordPiece tokenization. \n" "Sequences longer than this will be truncated, and sequences shorter \n" "than this will be padded.") parser.add_argument("--do_train", default=False, action='store_true', help="Whether to run training.") parser.add_argument("--do_eval", default=False, action='store_true', help="Whether to run eval on the dev set.") parser.add_argument("--train_batch_size", default=32, type=int, help="Total batch size for training.") parser.add_argument("--eval_batch_size", default=8, type=int, help="Total batch size for eval.") parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument("--num_train_epochs", default=3.0, type=float, help="Total number of training epochs to perform.") parser.add_argument("--warmup_proportion", default=0.1, type=float, help="Proportion of training to perform linear learning rate warmup for. " "E.g., 0.1 = 10%% of training.") parser.add_argument("--save_checkpoints_steps", default=1000, type=int, help="How often to save the model checkpoint.") parser.add_argument("--no_cuda", default=False, action='store_true', help="Whether not to use CUDA when available") parser.add_argument("--local_rank", type=int, default=-1, help="local_rank for distributed training on gpus") parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") parser.add_argument('--gradient_accumulation_steps', type=int, default=1, help="Number of updates steps to accumualte before performing a backward/update pass.") parser.add_argument('--optimize_on_cpu', default=False, action='store_true', help="Whether to perform optimization and keep the optimizer averages on CPU") parser.add_argument('--fp16', default=False, action='store_true', help="Whether to use 16-bit float precision instead of 32-bit") parser.add_argument('--loss_scale', type=float, default=128, help='Loss scaling, positive power of 2 values can improve fp16 convergence.') args = parser.parse_args() processors = { "cola": ColaProcessor, "mnli": MnliProcessor, "mrpc": MrpcProcessor, "news": NewsProcessor, } if args.local_rank == -1 or args.no_cuda: device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") n_gpu = torch.cuda.device_count() else: device = torch.device("cuda", args.local_rank) n_gpu = 1 # Initializes the distributed backend which will take care of sychronizing nodes/GPUs # torch.distributed.init_process_group(backend='nccl') if args.fp16: logger.info("16-bits training currently not supported in distributed training") args.fp16 = False # (see https://github.com/pytorch/pytorch/pull/13496) logger.info("device %s n_gpu %d distributed training %r", device, n_gpu, bool(args.local_rank != -1)) if args.gradient_accumulation_steps < 1: raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format( args.gradient_accumulation_steps)) args.train_batch_size = int(args.train_batch_size / args.gradient_accumulation_steps) random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if n_gpu > 0: torch.cuda.manual_seed_all(args.seed) if not args.do_train and not args.do_eval: raise ValueError("At least one of `do_train` or `do_eval` must be True.") bert_config = BertConfig.from_json_file(args.bert_config_file) if args.max_seq_length > bert_config.max_position_embeddings: raise ValueError( "Cannot use sequence length {} because the BERT model was only trained up to sequence length {}".format( args.max_seq_length, bert_config.max_position_embeddings)) if os.path.exists(args.output_dir) and os.listdir(args.output_dir): raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir)) os.makedirs(args.output_dir, exist_ok=True) task_name = args.task_name.lower() if task_name not in processors: raise ValueError("Task not found: %s" % (task_name)) processor = processors[task_name]() tokenizer = tokenization.FullTokenizer( vocab_file=args.vocab_file, do_lower_case=args.do_lower_case) train_examples = None num_train_steps = None if args.do_train: train_examples = processor.get_train_examples(args.data_dir) num_train_steps = int( len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps * args.num_train_epochs) label_list = processor.get_labels() print("label_list.size:%d\n" %(len(label_list))) # Prepare model model = BertForSequenceClassification(bert_config, len(label_list)) if args.init_checkpoint is not None: model.bert.load_state_dict(torch.load(args.init_checkpoint, map_location='cpu')) if args.fp16: model.half() model.to(device) #if args.local_rank != -1: #model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], # output_device=args.local_rank) #elif n_gpu > 1: # model = torch.nn.DataParallel(model) # Prepare optimizer if args.fp16: param_optimizer = [(n, param.clone().detach().to('cpu').float().requires_grad_()) \ for n, param in model.named_parameters()] elif args.optimize_on_cpu: param_optimizer = [(n, param.clone().detach().to('cpu').requires_grad_()) \ for n, param in model.named_parameters()] else: param_optimizer = list(model.named_parameters()) no_decay = ['bias', 'gamma', 'beta'] optimizer_grouped_parameters = [ {'params': [p for n, p in param_optimizer if n not in no_decay], 'weight_decay_rate': 0.01}, {'params': [p for n, p in param_optimizer if n in no_decay], 'weight_decay_rate': 0.0} ] optimizer = BERTAdam(optimizer_grouped_parameters, lr=args.learning_rate, warmup=args.warmup_proportion, t_total=num_train_steps) global_step = 0 if args.do_train: train_features = convert_examples_to_features( train_examples, label_list, args.max_seq_length, tokenizer) logger.info("***** Running training *****") logger.info(" Num examples = %d", len(train_examples)) logger.info(" Batch size = %d", args.train_batch_size) logger.info(" Num steps = %d", num_train_steps) all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long) all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long) train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) if args.local_rank == -1: train_sampler = RandomSampler(train_data) else: train_sampler = RandomSampler(train_data) #train_sampler = DistributedSampler(train_data) train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size) model.train() for _ in trange(int(args.num_train_epochs), desc="Epoch"): tr_loss = 0 nb_tr_examples, nb_tr_steps = 0, 0 for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration")): batch = tuple(t.to(device) for t in batch) input_ids, input_mask, segment_ids, label_ids = batch loss, _ = model(input_ids, segment_ids, input_mask, label_ids) if n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu. if args.fp16 and args.loss_scale != 1.0: # rescale loss for fp16 training # see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html loss = loss * args.loss_scale if args.gradient_accumulation_steps > 1: loss = loss / args.gradient_accumulation_steps loss.backward() tr_loss += loss.item() nb_tr_examples += input_ids.size(0) nb_tr_steps += 1 if (step + 1) % args.gradient_accumulation_steps == 0: if args.fp16 or args.optimize_on_cpu: if args.fp16 and args.loss_scale != 1.0: # scale down gradients for fp16 training for param in model.parameters(): param.grad.data = param.grad.data / args.loss_scale is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True) if is_nan: logger.info("FP16 TRAINING: Nan in gradients, reducing loss scaling") args.loss_scale = args.loss_scale / 2 model.zero_grad() continue optimizer.step() copy_optimizer_params_to_model(model.named_parameters(), param_optimizer) else: optimizer.step() model.zero_grad() global_step += 1 if args.do_eval: eval_examples = processor.get_dev_examples(args.data_dir) eval_features = convert_examples_to_features( eval_examples, label_list, args.max_seq_length, tokenizer) logger.info("***** Running evaluation *****") logger.info(" Num examples = %d", len(eval_examples)) logger.info(" Batch size = %d", args.eval_batch_size) all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long) all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long) eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) if args.local_rank == -1: eval_sampler = SequentialSampler(eval_data) else: eval_sampler = SequentialSampler(eval_data) #eval_sampler = DistributedSampler(eval_data) eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size) model.eval() eval_loss, eval_accuracy = 0, 0 nb_eval_steps, nb_eval_examples = 0, 0 for input_ids, input_mask, segment_ids, label_ids in eval_dataloader: input_ids = input_ids.to(device) input_mask = input_mask.to(device) segment_ids = segment_ids.to(device) label_ids = label_ids.to(device) with torch.no_grad(): tmp_eval_loss, logits = model(input_ids, segment_ids, input_mask, label_ids) logits = logits.detach().cpu().numpy() label_ids = label_ids.to('cpu').numpy() tmp_eval_accuracy = accuracy(logits, label_ids) eval_loss += tmp_eval_loss.mean().item() eval_accuracy += tmp_eval_accuracy nb_eval_examples += input_ids.size(0) nb_eval_steps += 1 eval_loss = eval_loss / nb_eval_steps eval_accuracy = eval_accuracy / nb_eval_examples result = {'eval_loss': eval_loss, 'eval_accuracy': eval_accuracy, 'global_step': global_step, 'loss': tr_loss/nb_tr_steps} output_eval_file = os.path.join(args.output_dir, "eval_results.txt") with open(output_eval_file, "w") as writer: logger.info("***** Eval results *****") for key in sorted(result.keys()): logger.info(" %s = %s", key, str(result[key])) writer.write("%s = %s\n" % (key, str(result[key])))
def main(): os.environ['CUDA_VISIBLE_DEVICES'] = '0' os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' root_path = r'./' bert_path = './chinese_L-12_H-768_A-12' flags = tf.flags flags.DEFINE_string("data_dir", os.path.join(root_path, 'data'), "The input datadir.", ) flags.DEFINE_string("bert_config_file", os.path.join(bert_path, 'bert_config.json'), "The config json file corresponding to the pre-trained BERT model.") flags.DEFINE_string("task_name", 'ner', "The name of the task to train.") flags.DEFINE_string("vocab_file", os.path.join(bert_path, 'vocab.txt'), "The vocabulary file that the BERT model was trained on.") flags.DEFINE_string("output_dir", os.path.join(root_path, 'model'), "The output directory where the model checkpoints will be written.") ## Other parameters flags.DEFINE_string("init_checkpoint", os.path.join(bert_path, 'pytorch_model.bin'), "Initial checkpoint (usually from a pre-trained BERT model).") flags.DEFINE_bool("do_lower_case", True, "Whether to lower case the input text.") flags.DEFINE_integer("max_seq_length", 48, "The maximum total input sequence length after WordPiece tokenization.") # flags.DEFINE_boolean('clean', True, 'remove the files which created by last training') flags.DEFINE_bool("do_train", True, "Whether to run training.") flags.DEFINE_bool("do_eval", True, "Whether to run eval on the dev set.") flags.DEFINE_bool("no_cuda", False, "Whether not to use CUDA when available") # flags.DEFINE_bool("do_predict", True, "Whether to run the model in inference mode on the test set.") flags.DEFINE_integer("train_batch_size", 64, "Total batch size for training.") flags.DEFINE_integer("eval_batch_size", 64, "Total batch size for eval.") # flags.DEFINE_integer("predict_batch_size", 4, "Total batch size for predict.") flags.DEFINE_float("learning_rate", 5e-5, "The initial learning rate for Adam.") flags.DEFINE_integer("save_model_epoch", 1, "save model ") flags.DEFINE_float("num_train_epochs", 15.0, "Total number of training epochs to perform.") flags.DEFINE_float('droupout_rate', 0.5, 'Dropout rate') flags.DEFINE_float('clip', 5, 'Gradient clip') flags.DEFINE_float("warmup_proportion", 0.1, "Proportion of training to perform linear learning rate warmup for. ""E.g., 0.1 = 10% of training.") flags.DEFINE_integer("save_checkpoints_steps", 50, "How often to save the model checkpoint.") flags.DEFINE_integer("iterations_per_loop", 50, "How many steps to make in each estimator call.") flags.DEFINE_integer("local_rank", -1, "local_rank for distributed training on gpus") flags.DEFINE_integer("seed", 1, "random seed for initialization") flags.DEFINE_integer("gradient_accumulation_steps", 1, "Number of updates steps to accumualte before performing a backward/update pass.") flags.DEFINE_bool("optimize_on_cpu", False, "Whether to perform optimization and keep the optimizer averages on CPU") flags.DEFINE_bool("fp16", False, "Whether to use 16-bit float precision instead of 32-bit") flags.DEFINE_float('loss_scale', 128.0, 'Loss scaling, positive power of 2 values can improve fp16 convergence.') args = flags.FLAGS if args.local_rank == -1 or args.no_cuda: device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") n_gpu = torch.cuda.device_count() else: device = torch.device("cuda", args.local_rank) n_gpu = 1 # Initializes the distributed backend which will take care of sychronizing nodes/GPUs # torch.distributed.init_process_group(backend='nccl') if args.fp16: logger.info("16-bits training currently not supported in distributed training") args.fp16 = False # (see https://github.com/pytorch/pytorch/pull/13496) logger.info("device %s n_gpu %d distributed training %r", device, n_gpu, bool(args.local_rank != -1)) if args.gradient_accumulation_steps < 1: raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format( args.gradient_accumulation_steps)) args.train_batch_size = int(args.train_batch_size / args.gradient_accumulation_steps) random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if n_gpu > 0: torch.cuda.manual_seed_all(args.seed) if not args.do_train and not args.do_eval: raise ValueError("At least one of `do_train` or `do_eval` must be True.") bert_config = BertConfig.from_json_file(args.bert_config_file) if args.max_seq_length > bert_config.max_position_embeddings: raise ValueError( "Cannot use sequence length {} because the BERT model was only trained up to sequence length {}".format( args.max_seq_length, bert_config.max_position_embeddings)) if os.path.exists(args.output_dir) and os.listdir(args.output_dir): raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir)) os.makedirs(args.output_dir, exist_ok=True) task_name = args.task_name.lower() processor = NewsProcessor() tokenizer = tokenization.FullTokenizer( vocab_file=args.vocab_file, do_lower_case=args.do_lower_case) train_examples = None num_train_steps = None if args.do_train: train_examples = processor.get_train_examples(args.data_dir) num_train_steps = int( len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps * args.num_train_epochs) if os.path.exists('./pkl/label_list.pkl'): label_list = load_pkl('label_list.pkl') if args.do_train and os.path.exists('./pkl/label_list.pkl') == False: label_list = processor.get_labels() save_pkl('label_list.pkl', label_list) print("label_list.size:%d\n" % (len(label_list))) # Prepare model model = BertForSequenceClassification(bert_config, len(label_list)) if args.init_checkpoint is not None: model.bert.load_state_dict(torch.load(args.init_checkpoint, map_location='cpu')) if args.fp16: model.half() model.to(device) # if args.local_rank != -1: # model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], # output_device=args.local_rank) # elif n_gpu > 1: # model = torch.nn.DataParallel(model) # Prepare optimizer if args.fp16: param_optimizer = [(n, param.clone().detach().to('cpu').float().requires_grad_()) \ for n, param in model.named_parameters()] elif args.optimize_on_cpu: param_optimizer = [(n, param.clone().detach().to('cpu').requires_grad_()) \ for n, param in model.named_parameters()] else: param_optimizer = list(model.named_parameters()) no_decay = ['bias', 'gamma', 'beta'] optimizer_grouped_parameters = [ {'params': [p for n, p in param_optimizer if n not in no_decay], 'weight_decay_rate': 0.01}, {'params': [p for n, p in param_optimizer if n in no_decay], 'weight_decay_rate': 0.0} ] optimizer = BERTAdam(optimizer_grouped_parameters, lr=args.learning_rate, warmup=args.warmup_proportion, t_total=num_train_steps) global_step = 0 if args.do_train: def get_dev_result(): eval_examples = processor.get_dev_examples(args.data_dir) eval_features = convert_examples_to_features(eval_examples, label_list, args.max_seq_length, tokenizer) logger.info("***** Running evaluation *****") logger.info(" Num examples = %d", len(eval_examples)) logger.info(" Batch size = %d", args.eval_batch_size) all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long) all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long) eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) eval_sampler = SequentialSampler(eval_data) eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size) model.eval() eval_loss, eval_accuracy = 0, 0 nb_eval_steps, nb_eval_examples = 0, 0 for input_ids, input_mask, segment_ids, label_ids in eval_dataloader: input_ids = input_ids.to(device) input_mask = input_mask.to(device) segment_ids = segment_ids.to(device) label_ids = label_ids.to(device) with torch.no_grad(): tmp_eval_loss, logits = model(input_ids, segment_ids, input_mask, label_ids) logits = logits.detach().cpu().numpy() label_ids = label_ids.to('cpu').numpy() tmp_eval_accuracy = accuracy(logits, label_ids) eval_loss += tmp_eval_loss.mean().item() eval_accuracy += tmp_eval_accuracy nb_eval_examples += input_ids.size(0) nb_eval_steps += 1 eval_accuracy = eval_accuracy / nb_eval_examples return eval_accuracy,eval_loss train_features = convert_examples_to_features(train_examples, label_list, args.max_seq_length, tokenizer) logger.info("***** Running training *****") logger.info(" Num examples = %d", len(train_examples)) logger.info(" Batch size = %d", args.train_batch_size) logger.info(" Num steps = %d", num_train_steps) all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long) all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long) train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) if args.local_rank == -1: train_sampler = RandomSampler(train_data) else: train_sampler = RandomSampler(train_data) # train_sampler = DistributedSampler(train_data) train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size) model.train() epoched = 1 for _ in trange(int(args.num_train_epochs), desc="Epoch"): tr_loss = 0 nb_tr_examples, nb_tr_steps = 0, 0 for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration")): batch = tuple(t.to(device) for t in batch) input_ids, input_mask, segment_ids, label_ids = batch loss, _ = model(input_ids, segment_ids, input_mask, label_ids) if n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu. if args.fp16 and args.loss_scale != 1.0: # rescale loss for fp16 training # see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html loss = loss * args.loss_scale if args.gradient_accumulation_steps > 1: loss = loss / args.gradient_accumulation_steps loss.backward() tr_loss += loss.item() nb_tr_examples += input_ids.size(0) nb_tr_steps += 1 if (step + 1) % args.gradient_accumulation_steps == 0: if args.fp16 or args.optimize_on_cpu: if args.fp16 and args.loss_scale != 1.0: # scale down gradients for fp16 training for param in model.parameters(): param.grad.data = param.grad.data / args.loss_scale is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True) if is_nan: logger.info("FP16 TRAINING: Nan in gradients, reducing loss scaling") args.loss_scale = args.loss_scale / 2 model.zero_grad() continue optimizer.step() copy_optimizer_params_to_model(model.named_parameters(), param_optimizer) else: optimizer.step() model.zero_grad() global_step += 1 acc,loss = get_dev_result() acc = round(acc,4) def write_file(filename, str): """ 写入文件 :param str: 字符串 :return: 无 """ writefile = open("./eval/" + filename, 'a+', encoding='utf-8') writefile.write(str + '\n') writefile.close() write_file('log.txt',str(epoched)+' acc: --> '+str(acc)) write_file('log.txt', str(epoched) + ' loss: --> ' + str(loss)) write_file('log.txt', '===========================') print('dev acc:', acc) print('loss',loss) if int(epoched)>2: if int(epoched) % int(args.save_model_epoch) == 0: # torch.save(model.state_dict(), args.output_dir + '/model_{}.pkl'.format(str(epoched))) torch.save(model, args.output_dir + '/model_{}.pkl'.format(str(epoched) + '_' + str(acc))) if int(epoched) == int(args.num_train_epochs): torch.save(model, args.output_dir + '/model_end.pkl') print('test acc:', acc) print('dev', loss) epoched += 1 if args.do_eval: eval_examples = processor.get_dev_examples(args.data_dir) eval_features = convert_examples_to_features(eval_examples, label_list, args.max_seq_length, tokenizer) logger.info("***** Running evaluation *****") logger.info(" Num examples = %d", len(eval_examples)) logger.info(" Batch size = %d", args.eval_batch_size) all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long) all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long) eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) eval_sampler = SequentialSampler(eval_data) eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size) if args.do_train == False: # model.load_state_dict(torch.load('./model/model_1.pkl', map_location='cpu')) model = torch.load('./model/model_end.pkl') model.eval() eval_loss, eval_accuracy = 0, 0 nb_eval_steps, nb_eval_examples = 0, 0 for input_ids, input_mask, segment_ids, label_ids in eval_dataloader: input_ids = input_ids.to(device) input_mask = input_mask.to(device) segment_ids = segment_ids.to(device) label_ids = label_ids.to(device) with torch.no_grad(): tmp_eval_loss, logits = model(input_ids, segment_ids, input_mask, label_ids) logits = logits.detach().cpu().numpy() label_ids = label_ids.to('cpu').numpy() tmp_eval_accuracy = accuracy(logits, label_ids) eval_loss += tmp_eval_loss.mean().item() eval_accuracy += tmp_eval_accuracy nb_eval_examples += input_ids.size(0) nb_eval_steps += 1 eval_loss = eval_loss / nb_eval_steps eval_accuracy = eval_accuracy / nb_eval_examples result = {'eval_loss': eval_loss, 'eval_accuracy': eval_accuracy, 'global_step': global_step, } output_eval_file = os.path.join("./eval", "eval_results.txt") with open(output_eval_file, "w") as writer: logger.info("***** Eval results *****") for key in sorted(result.keys()): logger.info(" %s = %s", key, str(result[key])) writer.write("%s = %s\n" % (key, str(result[key])))
def main(args): processors = { "cola": ColaProcessor, "mnli": MnliProcessor, "mrpc": MrpcProcessor, "news": NewsProcessor, } if args.local_rank == -1 or args.no_cuda: device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") n_gpu = torch.cuda.device_count() else: device = torch.device("cuda", args.local_rank) n_gpu = 1 # Initializes the distributed backend which will take care of sychronizing nodes/GPUs # torch.distributed.init_process_group(backend='nccl') if args.fp16: logger.info( "16-bits training currently not supported in distributed training" ) args.fp16 = False # (see https://github.com/pytorch/pytorch/pull/13496) logger.info("device %s n_gpu %d distributed training %r", device, n_gpu, bool(args.local_rank != -1)) if args.gradient_accumulation_steps < 1: raise ValueError( "Invalid gradient_accumulation_steps parameter: {}, should be >= 1" .format(args.gradient_accumulation_steps)) args.train_batch_size = int(args.train_batch_size / args.gradient_accumulation_steps) random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if n_gpu > 0: torch.cuda.manual_seed_all(args.seed) if not args.do_train and not args.do_eval: raise ValueError( "At least one of `do_train` or `do_eval` must be True.") # bert_config = BertConfig.from_json_file(args.bert_config_file) bert_config = BertConfig(args.vocab_size) if args.max_seq_length > bert_config.max_position_embeddings: raise ValueError( "Cannot use sequence length {} because the BERT model was only trained up to sequence length {}" .format(args.max_seq_length, bert_config.max_position_embeddings)) if os.path.exists(args.output_dir) and os.listdir(args.output_dir): raise ValueError( "Output directory ({}) already exists and is not empty.".format( args.output_dir)) os.makedirs(args.output_dir, exist_ok=True) task_name = args.task_name.lower() if task_name not in processors: raise ValueError("Task not found: %s" % (task_name)) processor = processors[task_name]() tokenizer = tokenization.FullTokenizer(vocab_file=args.vocab_file, do_lower_case=args.do_lower_case) train_examples = None num_train_steps = None if args.do_train: train_examples = processor.get_train_examples(args.data_dir) num_train_steps = int( len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps * args.num_train_epochs) label_list = processor.get_labels() print("label_list.size:%d\n" % (len(label_list))) # Prepare model model = BertForSequenceClassification(bert_config, len(label_list)) if args.init_checkpoint is not None: model.bert.load_state_dict( torch.load(args.init_checkpoint, map_location='cpu')) if args.fp16: model.half() model.to(device) # if args.local_rank != -1: # model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], # output_device=args.local_rank) # elif n_gpu > 1: # model = torch.nn.DataParallel(model) # Prepare optimizer if args.fp16: param_optimizer = [(n, param.clone().detach().to('cpu').float().requires_grad_()) \ for n, param in model.named_parameters()] elif args.optimize_on_cpu: param_optimizer = [(n, param.clone().detach().to('cpu').requires_grad_()) \ for n, param in model.named_parameters()] else: param_optimizer = list(model.named_parameters()) no_decay = ['bias', 'gamma', 'beta'] optimizer_grouped_parameters = [{ 'params': [p for n, p in param_optimizer if n not in no_decay], 'weight_decay_rate': 0.01 }, { 'params': [p for n, p in param_optimizer if n in no_decay], 'weight_decay_rate': 0.0 }] optimizer = BERTAdam(optimizer_grouped_parameters, lr=args.learning_rate, warmup=args.warmup_proportion, t_total=num_train_steps) global_step = 0 if args.do_train: train_features = convert_examples_to_features(train_examples, label_list, args.max_seq_length, tokenizer) logger.info("***** Running training *****") logger.info(" Num examples = %d", len(train_examples)) logger.info(" Batch size = %d", args.train_batch_size) logger.info(" Num steps = %d", num_train_steps) all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long) all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long) train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) if args.local_rank == -1: train_sampler = RandomSampler(train_data) else: train_sampler = RandomSampler(train_data) # train_sampler = DistributedSampler(train_data) train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size) model.train() ### 状态设置 for _ in trange(int(args.num_train_epochs), desc="Epoch"): tr_loss = 0 nb_tr_examples, nb_tr_steps = 0, 0 for step, batch in enumerate( tqdm(train_dataloader, desc="Iteration")): batch = tuple(t.to(device) for t in batch) input_ids, input_mask, segment_ids, label_ids = batch loss, _ = model(input_ids, segment_ids, input_mask, label_ids) if n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu. if args.fp16 and args.loss_scale != 1.0: # rescale loss for fp16 training # see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html loss = loss * args.loss_scale if args.gradient_accumulation_steps > 1: loss = loss / args.gradient_accumulation_steps loss.backward() tr_loss += loss.item() nb_tr_examples += input_ids.size(0) nb_tr_steps += 1 if (step + 1) % args.gradient_accumulation_steps == 0: if args.fp16 or args.optimize_on_cpu: if args.fp16 and args.loss_scale != 1.0: # scale down gradients for fp16 training for param in model.parameters(): param.grad.data = param.grad.data / args.loss_scale is_nan = set_optimizer_params_grad( param_optimizer, model.named_parameters(), test_nan=True) if is_nan: logger.info( "FP16 TRAINING: Nan in gradients, reducing loss scaling" ) args.loss_scale = args.loss_scale / 2 model.zero_grad() continue optimizer.step() copy_optimizer_params_to_model( model.named_parameters(), param_optimizer) else: optimizer.step() model.zero_grad() global_step += 1 if args.do_eval: eval_examples = processor.get_dev_examples(args.data_dir) eval_features = convert_examples_to_features(eval_examples, label_list, args.max_seq_length, tokenizer) logger.info("***** Running evaluation *****") logger.info(" Num examples = %d", len(eval_examples)) logger.info(" Batch size = %d", args.eval_batch_size) all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long) all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long) eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) if args.local_rank == -1: eval_sampler = SequentialSampler(eval_data) else: eval_sampler = SequentialSampler(eval_data) # eval_sampler = DistributedSampler(eval_data) eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size) model.eval() eval_loss, eval_accuracy = 0, 0 nb_eval_steps, nb_eval_examples = 0, 0 for input_ids, input_mask, segment_ids, label_ids in eval_dataloader: input_ids = input_ids.to(device) input_mask = input_mask.to(device) segment_ids = segment_ids.to(device) label_ids = label_ids.to(device) with torch.no_grad(): tmp_eval_loss, logits = model(input_ids, segment_ids, input_mask, label_ids) logits = logits.detach().cpu().numpy() label_ids = label_ids.to('cpu').numpy() tmp_eval_accuracy = accuracy(logits, label_ids) eval_loss += tmp_eval_loss.mean().item() eval_accuracy += tmp_eval_accuracy nb_eval_examples += input_ids.size(0) nb_eval_steps += 1 eval_loss = eval_loss / nb_eval_steps eval_accuracy = eval_accuracy / nb_eval_examples result = { 'eval_loss': eval_loss, 'eval_accuracy': eval_accuracy, 'global_step': global_step, 'loss': tr_loss / nb_tr_steps } output_eval_file = os.path.join(args.output_dir, "eval_results.txt") with open(output_eval_file, "w") as writer: logger.info("***** Eval results *****") for key in sorted(result.keys()): logger.info(" %s = %s", key, str(result[key])) writer.write("%s = %s\n" % (key, str(result[key]))) torch.save(model.state_dict(), args.output_dir + '/bert_' + args.task_name + 'pt')
def main(): parser = argparse.ArgumentParser() ## Required parameters parser.add_argument("--data_dir", default=r"C:\workspace\python-work\nlp\nlp_framework\nlp-tutorial-ec\bert-Chinese-classification-task-ec\glue_data\data", type=str, # required=True, help="The input data dir. Should contain the .tsv files (or other data files) for the task.") parser.add_argument("--bert_config_file", default=r"C:\workspace\python-work\nlp\nlp_framework\nlp-tutorial-ec\bert-Chinese-classification-task-ec\config\bert_config.json", type=str, # required=True, help="The config json file corresponding to the pre-trained BERT model. \n" "This specifies the model architecture.") parser.add_argument("--task_name", default="NEWS", type=str, # required=True, help="The name of the task to train.") parser.add_argument("--vocab_file", default=r"C:\workspace\python-work\nlp\nlp_framework\nlp-tutorial-ec\bert-Chinese-classification-task-ec\config\vocab.txt", type=str, # required=True, help="The vocabulary file that the BERT model was trained on.") parser.add_argument("--output_dir", default="./newsAll_output/", type=str, # required=True, help="The output directory where the model checkpoints will be written.") ## Other parameters parser.add_argument("--init_checkpoint", default=r"D:\database\bert\chinese_L-12_H-768_A-12\pytorch_model.bin", type=str, help="Initial checkpoint (usually from a pre-trained BERT model).") parser.add_argument("--do_lower_case", default=False, action='store_true', help="Whether to lower case the input text. True for uncased models, False for cased models.") parser.add_argument("--max_seq_length", default=256, type=int, help="The maximum total input sequence length after WordPiece tokenization. \n" "Sequences longer than this will be truncated, and sequences shorter \n" "than this will be padded.") parser.add_argument("--do_train", default=True, action='store_true', help="Whether to run training.") parser.add_argument("--do_eval", default=True, action='store_true', help="Whether to run eval on the dev set.") parser.add_argument("--train_batch_size", default=8, type=int, help="Total batch size for training.") parser.add_argument("--eval_batch_size", default=8, type=int, help="Total batch size for eval.") parser.add_argument("--learning_rate", default=2e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument("--num_train_epochs", default=1.0, type=float, help="total number of training epochs to perform.") parser.add_argument("--warmup_proportion", default=0.1, type=float, help="Proportion of training to perform linear learning rate warmup for. " "E.g., 0.1 = 10%% of training.") parser.add_argument("--save_checkpoints_steps", default=1000, type=int, help="How often to save the model checkpoint.") parser.add_argument("--no_cuda", default=True, action='store_true', help="Whether not to use CUDA when available") parser.add_argument("--local_rank", type=int, default=-1, help="local_rank for distributed training on gpus") parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") parser.add_argument('--gradient_accumulation_steps', type=int, default=1, help="Number of updates steps to accumualte before performing a backward/update pass.") parser.add_argument('--optimize_on_cpu', default=False, action='store_true', help="Whether to perform optimization and keep the optimizer averages on CPU") parser.add_argument('--fp16', default=False, action='store_true', help="Whether to use 16-bit float precision instead of 32-bit") parser.add_argument('--loss_scale', type=float, default=128, help='Loss scaling, positive power of 2 values can improve fp16 convergence.') args = parser.parse_args() # opt=get_config(vars(args)) print("args---------->", args, "\n") processors = { "cola": ColaProcessor, "mnli": MnliProcessor, "mrpc": MrpcProcessor, "news": NewsProcessor, } if args.local_rank == -1 or args.no_cuda: # device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") device = torch.device("cpu") n_gpu = 0 # n_gpu = torch.cuda.device_count() print("device", device) else: device = torch.device("cuda", args.local_rank) n_gpu = 1 # Initializes the distributed backend which will take care of sychronizing nodes/GPUs # torch.distributed.init_process_group(backend='nccl') if args.fp16: logger.info("16-bits training currently not supported in distributed training") args.fp16 = False # (see https://github.com/pytorch/pytorch/pull/13496) logger.info("device %s n_gpu %d distributed training %r", device, n_gpu, bool(args.local_rank != -1)) if args.gradient_accumulation_steps < 1: raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format( args.gradient_accumulation_steps)) args.train_batch_size = int(args.train_batch_size / args.gradient_accumulation_steps) random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if n_gpu > 0: torch.cuda.manual_seed_all(args.seed) if not args.do_train and not args.do_eval: raise ValueError("At least one of `do_train` or `do_eval` must be True.") bert_config = BertConfig.from_json_file(args.bert_config_file) if args.max_seq_length > bert_config.max_position_embeddings: raise ValueError( "Cannot use sequence length {} because the BERT model was only trained up to sequence length {}".format( args.max_seq_length, bert_config.max_position_embeddings)) if os.path.exists(args.output_dir) and os.listdir(args.output_dir): raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir)) os.makedirs(args.output_dir, exist_ok=True) task_name = args.task_name.lower() if task_name not in processors: raise ValueError("Task not found: %s" % (task_name)) # processor = NewsProcessor() processor = processors[task_name]() # print("tmp-----",processor) tokenizer = tokenization.FullTokenizer( vocab_file=args.vocab_file, do_lower_case=args.do_lower_case) train_examples = None num_train_steps = None if args.do_train: # train_examples: examples.append( InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) # NewsProcessor.get_train_examples(args.data_dir) train_examples = processor.get_train_examples(args.data_dir) # 300/8/1 * 1 = 37 num_train_steps = int( len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps * args.num_train_epochs) label_list = processor.get_labels() # <class 'list'>: ['houseliving', 'game', 'fashion'] print("label_list.size:%d\n" % (len(label_list))) # ------------------------------------important----------------------------------------- # ------------------------------------important----------------------------------------- # ------------------------------------important----------------------------------------- # ------------------------------------important----------------------------------------- # model # Prepare model #label_list <class 'list'>: ['fashion', 'game', 'houseliving'] model = BertForSequenceClassification(bert_config, len(label_list)) #3 # print(model) ''' BertForSequenceClassification( (bert): BertModel( (embeddings): BERTEmbeddings( (word_embeddings): Embedding(21128, 768) (position_embeddings): Embedding(512, 768) (token_type_embeddings): Embedding(2, 768) (LayerNorm): BERTLayerNorm() (dropout): Dropout(p=0.1) ) (encoder): BERTEncoder( (layer): ModuleList( (0-11): BERTLayer( (attention): BERTAttention( (self): BERTSelfAttention( (query): Linear(in_features=768, out_features=768, bias=True) (key): Linear(in_features=768, out_features=768, bias=True) (value): Linear(in_features=768, out_features=768, bias=True) (dropout): Dropout(p=0.1) ) (output): BERTSelfOutput( (dense): Linear(in_features=768, out_features=768, bias=True) (LayerNorm): BERTLayerNorm() (dropout): Dropout(p=0.1) ) ) (intermediate): BERTIntermediate( (dense): Linear(in_features=768, out_features=3072, bias=True) ) (output): BERTOutput( (dense): Linear(in_features=3072, out_features=768, bias=True) (LayerNorm): BERTLayerNorm() (dropout): Dropout(p=0.1) ) ) ) ) (pooler): BERTPooler( (dense): Linear(in_features=768, out_features=768, bias=True) (activation): Tanh() ) ) (dropout): Dropout(p=0.1) (classifier): Linear(in_features=768, out_features=3, bias=True) ) ''' # BertModel loading trained parameters if args.init_checkpoint is not None: model.bert.load_state_dict(torch.load(args.init_checkpoint, map_location='cpu')) if args.fp16: model.half() model.to(device) # if args.local_rank != -1: # model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], # output_device=args.local_rank) # elif n_gpu > 1: # model = torch.nn.DataParallel(model) # Prepare optimizer if args.fp16: param_optimizer = [(n, param.clone().detach().to('cpu').float().requires_grad_()) \ for n, param in model.named_parameters()] elif args.optimize_on_cpu: param_optimizer = [(n, param.clone().detach().to('cpu').requires_grad_()) \ for n, param in model.named_parameters()] else: # loaded parameter ----------->list[len:12layer] name:parameter param_optimizer = list(model.named_parameters()) no_decay = ['bias', 'gamma', 'beta'] # len(optimizer_grouped_parameters)=2 # optimizer_grouped_parameters[0]:dict-----> optimizer_grouped_parameters[0]['weight_decay_rate'] =0.01 optimizer_grouped_parameters = [ # for name,parameter inparm_optimizer if name not in no_dacay {'params': [p for n, p in param_optimizer if n not in no_decay], 'weight_decay_rate': 0.01}, {'params': [p for n, p in param_optimizer if n in no_decay], 'weight_decay_rate': 0.0} ] optimizer = BERTAdam(optimizer_grouped_parameters, lr=args.learning_rate, warmup=args.warmup_proportion, t_total=num_train_steps) global_step = 0 if args.do_train: #这个操作中没有masked language model 只有token——embedding 和segment——embedding # train_features: features.append( InputFeatures( input_ids=input_ids, input_mask=input_mask, segment_ids=segment_ids, label_id=label_id)) train_features = convert_examples_to_features(train_examples, label_list, args.max_seq_length, tokenizer) logger.info("***** Running training *****") logger.info(" Num examples = %d", len(train_examples)) logger.info(" Batch size = %d", args.train_batch_size) logger.info(" Num steps = %d", num_train_steps) all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long) all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long) train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) if args.local_rank == -1: train_sampler = RandomSampler(train_data) else: train_sampler = RandomSampler(train_data) # train_sampler = DistributedSampler(train_data) train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size) model.train() for _ in trange(int(args.num_train_epochs), desc="Epoch"): tr_loss = 0 nb_tr_examples, nb_tr_steps = 0, 0 for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration")): if(step<1): # print(2,type(batch)) #list batch = tuple(t.to(device) for t in batch) input_ids, input_mask, segment_ids, label_ids = batch ####################################################################################\ # -----------------------training--------------------------------------- # -----------------------training--------------------------------------- # model 里面有torch.nn.CrossEntropyLoss(), 返回loss loss, _ = model(input_ids, segment_ids, input_mask, label_ids) if n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu. if args.fp16 and args.loss_scale != 1.0: # rescale loss for fp16 training # see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html loss = loss * args.loss_scale if args.gradient_accumulation_steps > 1: loss = loss / args.gradient_accumulation_steps loss.backward() tr_loss += loss.item() nb_tr_examples += input_ids.size(0) #=b=8 nb_tr_steps += 1 #0+1=1 if (step + 1) % args.gradient_accumulation_steps == 0: #gradient_accumulation_steps=1 if args.fp16 or args.optimize_on_cpu: if args.fp16 and args.loss_scale != 1.0: # scale down gradients for fp16 training for param in model.parameters(): param.grad.data = param.grad.data / args.loss_scale is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True) if is_nan: logger.info("FP16 TRAINING: Nan in gradients, reducing loss scaling") args.loss_scale = args.loss_scale / 2 model.zero_grad() continue optimizer.step() copy_optimizer_params_to_model(model.named_parameters(), param_optimizer) else: optimizer.step() model.zero_grad() global_step += 1 if args.do_eval: eval_examples = processor.get_dev_examples(args.data_dir) eval_features = convert_examples_to_features( eval_examples, label_list, args.max_seq_length, tokenizer) logger.info("***** Running evaluation *****") logger.info(" Num examples = %d", len(eval_examples)) logger.info(" Batch size = %d", args.eval_batch_size) all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long) all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long) eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) if args.local_rank == -1: eval_sampler = SequentialSampler(eval_data) else: eval_sampler = SequentialSampler(eval_data) # eval_sampler = DistributedSampler(eval_data) eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size) model.eval() eval_loss, eval_accuracy = 0, 0 nb_eval_steps, nb_eval_examples = 0, 0 for input_ids, input_mask, segment_ids, label_ids in eval_dataloader: input_ids = input_ids.to(device) input_mask = input_mask.to(device) segment_ids = segment_ids.to(device) label_ids = label_ids.to(device) with torch.no_grad(): #logits: (b,output_size)=(8,3) 表示最后一个encode输出的(b,m,d)中的firt token ---->(b,d)---------->(b,ouputsize) tmp_eval_loss, logits = model(input_ids, segment_ids, input_mask, label_ids) logits = logits.detach().cpu().numpy() label_ids = label_ids.to('cpu').numpy() tmp_eval_accuracy = accuracy(logits, label_ids) #每一个batch预测对的个数=4 eval_loss += tmp_eval_loss.mean().item() eval_accuracy += tmp_eval_accuracy nb_eval_examples += input_ids.size(0) nb_eval_steps += 1 eval_loss = eval_loss / nb_eval_steps #所有次数loss的平均值 eval_accuracy = eval_accuracy / nb_eval_examples #预测对的/所有的预测数目 result = {'eval_loss': eval_loss, 'eval_accuracy': eval_accuracy, 'global_step': global_step, 'loss': tr_loss / nb_tr_steps} #training中的loss平均值 output_eval_file = os.path.join(args.output_dir, "eval_results.txt") #./newsAll_output/ eval_results.txt with open(output_eval_file, "w") as writer: logger.info("***** Eval results *****") for key in sorted(result.keys()): logger.info(" %s = %s", key, str(result[key])) writer.write("%s = %s\n" % (key, str(result[key])))