def main(): parser = argparse.ArgumentParser() parser.add_argument("--checkpoint", default='output/korquad_3.bin', type=str, help="checkpoint") parser.add_argument( "--output_dir", default='debug', type=str, help= "The output directory where the model checkpoints and predictions will be written." ) ## Other parameters parser.add_argument( "--predict_file", default='data/KorQuAD_v1.0_dev.json', type=str, help="SQuAD json for predictions. E.g., dev-v1.1.json or test-v1.1.json" ) parser.add_argument( "--config_name", default="data/bert_small.json", type=str, help="Pretrained config name or path if not the same as model_name") parser.add_argument( "--max_seq_length", default=512, type=int, help= "The maximum total input sequence length after WordPiece tokenization. Sequences " "longer than this will be truncated, and sequences shorter than this will be padded." ) parser.add_argument( "--doc_stride", default=64, type=int, help= "When splitting up a long document into chunks, how much stride to take between chunks." ) parser.add_argument( "--max_query_length", default=96, type=int, help= "The maximum number of tokens for the question. Questions longer than this will " "be truncated to this length.") parser.add_argument( "--do_lower_case", action='store_true', help="Set this flag if you are using an uncased model.") parser.add_argument("--batch_size", default=16, type=int, help="Batch size per GPU/CPU for evaluation.") parser.add_argument( "--n_best_size", default=20, type=int, help= "The total number of n-best predictions to generate in the nbest_predictions.json output file." ) parser.add_argument( "--max_answer_length", default=30, type=int, help= "The maximum length of an answer that can be generated. This is needed because the start " "and end predictions are not conditioned on one another.") parser.add_argument( "--verbose_logging", action='store_true', help= "If true, all of the warnings related to data processing will be printed. " "A number of warnings are expected for a normal SQuAD evaluation.") parser.add_argument("--no_cuda", action='store_true', help="Whether not to use CUDA when available") parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") parser.add_argument( '--fp16', action='store_true', help= "Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit" ) args = parser.parse_args() # Setup CUDA, GPU & distributed training device = torch.device( "cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") args.n_gpu = torch.cuda.device_count() args.device = device # Setup logging logging.basicConfig( format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO) logger.info("device: %s, n_gpu: %s, 16-bits training: %s", device, args.n_gpu, args.fp16) # Set seed set_seed(args) tokenizer = BertTokenizer(vocab_file='data/wiki_vocab_32k.txt', do_basic_tokenize=True, max_len=args.max_seq_length) config = Config.from_json_file(args.config_name) model = QuestionAnswering(config) model.load_state_dict(torch.load(args.checkpoint)) num_params = count_parameters(model) logger.info("Total Parameter: %d" % num_params) if args.fp16: model.half() model.to(args.device) logger.info("Evaluation parameters %s", args) # Evaluate examples, features = load_and_cache_examples(args, tokenizer) evaluate(args, model, examples, features)
# Setup logging logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt = '%m/%d/%Y %H:%M:%S', level = logging.INFO if args.local_rank in [-1, 0] else logging.WARN) logger.warning("Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s", args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16) # Set seed set_seed(args) tokenizer = BertTokenizer(args.vocab_file, max_len=args.max_seq_length, do_basic_tokenize=True) # Prepare model config = Config.from_json_file(args.config_file) model = QuestionAnswering(config) model.bert.load_state_dict(torch.load(args.checkpoint)) num_params = count_parameters(model) logger.info("Total Parameter: %d" % num_params) model.to(device) logger.info("Training hyper-parameters %s", args) # Training train_dataset = load_and_cache_examples(args, tokenizer) train(args, train_dataset, model) if __name__ == "__main__": main()
def main(): parser = argparse.ArgumentParser() # Parameters parser.add_argument( "--output_dir", default='output', type=str, help= "The output directory where the model checkpoints and predictions will be written." ) parser.add_argument("--checkpoint", default='large_v1/large_v1_model.bin', type=str, help="pre-trained model checkpoint") parser.add_argument("--config_file", default='large_v1/large_config.json', type=str, help="model configuration file") parser.add_argument("--vocab_file", default='large_v1/large_v1_32k_vocab.txt', type=str, help="tokenizer vocab file") parser.add_argument("--train_file", default='data/korquad/KorQuAD_v1.0_train.json', type=str, help="SQuAD json for training. E.g., train-v1.1.json") parser.add_argument("--test", default=False, type=bool, help="Test by sample") parser.add_argument("--train_data_limit", default=-1, type=int, help="For Check EDA performance limit train data") ######ADD for EDA####### parser.add_argument( "--same_random_index", default=True, type=bool, help="for eda data limit, if True every eda pick same random index") parser.add_argument("--eda_type", default="no_eda", type=str, help="eda type e.g) sr, ri, rs, rd") parser.add_argument("--alpha", default=0.1, type=float, help="For eda") parser.add_argument("--num_aug", default=2, type=int, help="For eda") parser.add_argument("--eda_model_name", default="word2vec", type=str, help="For eda") parser.add_argument("--min_score", default=0.7, type=float, help="For eda") parser.add_argument("--dict_file_name", default="data/syn/synonyms_file.pickle", type=str) ######################## parser.add_argument( "--max_seq_length", default=512, type=int, help= "The maximum total input sequence length after WordPiece tokenization. Sequences " "longer than this will be truncated, and sequences shorter than this will be padded." ) parser.add_argument( "--doc_stride", default=128, type=int, help= "When splitting up a long document into chunks, how much stride to take between chunks." ) parser.add_argument( "--max_query_length", default=64, type=int, help= "The maximum number of tokens for the question. Questions longer than this will " "be truncated to this length.") parser.add_argument( "--max_answer_length", default=30, type=int, help= "The maximum length of an answer that can be generated. This is needed because the start " "and end predictions are not conditioned on one another.") parser.add_argument("--per_gpu_train_batch_size", default=4, type=int, help="Total batch size for training.") parser.add_argument( '--gradient_accumulation_steps', default=1, type=int, help= "Number of updates steps to accumulate before performing a backward/update pass." ) parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument("--weight_decay", default=0.01, type=float, help="Weight deay if we apply some.") parser.add_argument("--num_train_epochs", default=4.0, type=float, help="Total number of training epochs to perform.") parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") parser.add_argument("--adam_epsilon", default=1e-6, type=float, help="Epsilon for Adam optimizer.") parser.add_argument( "--warmup_proportion", default=0.1, type=float, help= "Proportion of training to perform linear learning rate warmup for. E.g., 0.1 = 10%% " "of training.") parser.add_argument( "--n_best_size", default=20, type=int, help= "The total number of n-best predictions to generate in the nbest_predictions.json " "output file.") parser.add_argument( "--verbose_logging", action='store_true', help= "If true, all of the warnings related to data processing will be printed. " "A number of warnings are expected for a normal SQuAD evaluation.") parser.add_argument("--no_cuda", action='store_true', help="Whether not to use CUDA when available") parser.add_argument('--seed', default=42, type=int, help="random seed for initialization") parser.add_argument( '--fp16', action='store_true', help="Whether to use 16-bit float precision instead of 32-bit") parser.add_argument( '--fp16_opt_level', default='O2', type=str, help= "For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']." "See details at https://nvidia.github.io/apex/amp.html") parser.add_argument("--local_rank", default=-1, type=int, help="For distributed training: local_rank") parser.add_argument( '--null_score_diff_threshold', type=float, default=0.0, help= "If null_score - best_non_null is greater than the threshold predict null." ) args = parser.parse_args() # Setup CUDA, GPU & distributed training if args.local_rank == -1 or args.no_cuda: device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") args.n_gpu = torch.cuda.device_count() else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs torch.cuda.set_device(args.local_rank) device = torch.device("cuda", args.local_rank) torch.distributed.init_process_group(backend='nccl') args.n_gpu = 1 args.device = device # Setup logging logging.basicConfig( format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN) logger.warning( "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s", args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16) # Set seed set_seed(args) tokenizer = BertTokenizer(args.vocab_file, max_len=args.max_seq_length, do_basic_tokenize=True) # Prepare model config = Config.from_json_file(args.config_file) model = QuestionAnswering(config) model.load_state_dict(torch.load(args.checkpoint)) num_params = count_parameters(model) logger.info("Total Parameter: %d" % num_params) model.to(device) logger.info("Training hyper-parameters %s", args) # Training train_dataset = load_and_cache_examples(args, tokenizer) train(args, train_dataset, model)
def main(): parser = argparse.ArgumentParser() parser.add_argument("--checkpoint", default='output/korquad_3.bin', type=str, help="checkpoint") parser.add_argument( "--output_dir", default='debug', type=str, help= "The output directory where the model checkpoints and predictions will be written." ) parser.add_argument("--model_config", type=str) parser.add_argument("--vocab", type=str) ## Other parameters parser.add_argument( "--predict_file", default='data/KorQuAD_v1.0_dev.json', type=str, help="SQuAD json for predictions. E.g., dev-v1.1.json or test-v1.1.json" ) parser.add_argument( "--max_seq_length", default=512, type=int, help= "The maximum total input sequence length after WordPiece tokenization. Sequences " "longer than this will be truncated, and sequences shorter than this will be padded." ) parser.add_argument( "--doc_stride", default=64, type=int, help= "When splitting up a long document into chunks, how much stride to take between chunks." ) parser.add_argument( "--max_query_length", default=96, type=int, help= "The maximum number of tokens for the question. Questions longer than this will " "be truncated to this length.") parser.add_argument( "--do_lower_case", action='store_true', help="Set this flag if you are using an uncased model.") parser.add_argument("--batch_size", default=16, type=int, help="Batch size per GPU/CPU for evaluation.") parser.add_argument( "--n_best_size", default=20, type=int, help= "The total number of n-best predictions to generate in the nbest_predictions.json output file." ) parser.add_argument( "--max_answer_length", default=30, type=int, help= "The maximum length of an answer that can be generated. This is needed because the start " "and end predictions are not conditioned on one another.") parser.add_argument( "--verbose_logging", action='store_true', help= "If true, all of the warnings related to data processing will be printed. " "A number of warnings are expected for a normal SQuAD evaluation.") parser.add_argument("--no_cuda", action='store_true', help="Whether not to use CUDA when available") parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") parser.add_argument( '--fp16', action='store_true', help= "Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit" ) parser.add_argument('--ensemble', default='false', type=str) args = parser.parse_args() args.ensemble = args.ensemble.lower() == 'true' # Setup CUDA, GPU & distributed training device = torch.device( "cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") args.n_gpu = torch.cuda.device_count() args.device = device # Setup logging logging.basicConfig( format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO) logger.info("device: %s, n_gpu: %s, 16-bits training: %s", device, args.n_gpu, args.fp16) # Set seed set_seed(args) tokenizer = BertTokenizer(vocab_file=args.vocab, do_basic_tokenize=True, max_len=args.max_seq_length) config = Config.from_json_file(args.model_config) model = QuestionAnswering(config) # Evaluate examples, features = load_and_cache_examples(args, tokenizer) if not args.ensemble: logger.info(" Load Model: %s ", args.checkpoint) model.load_state_dict(torch.load(args.checkpoint)) num_params = count_parameters(model) logger.info("Total Parameter: %d" % num_params) if args.fp16: model.half() model.to(args.device) logger.info("Evaluation parameters %s", args) results = evaluate(args, model, examples, features) else: list_ckpts = [] with open(os.path.join(args.output_dir, "ckpt_list.txt"), 'r') as f: for line in f: list_ckpts.append(line.strip()) list_results = [] for i, ckpt in enumerate(list_ckpts): fn = os.path.join(args.output_dir, ckpt) logger.info(" Load Model: %s ", fn) model.load_state_dict(torch.load(fn)) num_params = count_parameters(model) logger.info("Total Parameter: %d" % num_params) if args.fp16: model.half() model.to(args.device) logger.info("Evaluation parameters %s", args) results = evaluate(args, model, examples, features) list_results.append(results) list_qid = [] for example in examples: list_qid.append(example.qas_id) all_predictions = collections.OrderedDict() for qid in list_qid: max_prob, answer = 0.0, "" for results in list_results: prob, text = 0.0, None for output in results[qid]: if output["text"]: prob = output["probability"] text = output["text"] break if prob > max_prob: max_prob = prob answer = text all_predictions[qid] = answer output_prediction_file = os.path.join(args.output_dir, "predictions.json") with open(output_prediction_file, "w") as writer: writer.write(json.dumps(all_predictions, indent=4) + "\n") expected_version = 'KorQuAD_v1.0' with open(args.predict_file) as dataset_file: dataset_json = json.load(dataset_file) read_version = "_".join(dataset_json['version'].split("_")[:-1]) if (read_version != expected_version): logger.info('Evaluation expects ' + expected_version + ', but got dataset with ' + read_version, file=sys.stderr) dataset = dataset_json['data'] with open(os.path.join(args.output_dir, "predictions.json")) as prediction_file: predictions = json.load(prediction_file) logger.info(json.dumps(korquad_eval(dataset, predictions)))
def main(): parser = argparse.ArgumentParser() # Required Parameters parser.add_argument( "--output_dir", default='output', type=str, help= "The output directory where the model checkpoints and predictions will be written." ) parser.add_argument("--checkpoint", default='pretrain_ckpt/bert_small_ckpt.bin', type=str, help="checkpoint") parser.add_argument("--resume_checkpoint", default=False, type=bool, help="resume") parser.add_argument('--log_dir', default='./runs', type=str) parser.add_argument("--model_config", default='data/bert_small.json', type=str) # Other Parameters parser.add_argument( "--corpus", default='./rsc/corpus/korquad_corpus_post_training.hdf5', type=str, help="SQuAD corpus for post-training.") parser.add_argument( "--max_seq_length", default=512, type=int, help= "The maximum total input sequence length after WordPiece tokenization. Sequences " "longer than this will be truncated, and sequences shorter than this will be padded." ) parser.add_argument("--train_batch_size", default=16, type=int, help="Total batch size for training.") parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument("--num_train_epochs", default=4.0, type=float, help="Total number of training epochs to perform.") parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") parser.add_argument("--adam_epsilon", default=1e-6, type=float, help="Epsilon for Adam optimizer.") parser.add_argument( "--warmup_proportion", default=0.1, type=float, help= "Proportion of training to perform linear learning rate warmup for. E.g., 0.1 = 10%% " "of training.") parser.add_argument("--num_workers", default=8, type=int, help="Proportion of workers of DataLoader") parser.add_argument("--no_cuda", action='store_true', help="Whether not to use CUDA when available") parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") parser.add_argument( '--fp16', action='store_true', help="Whether to use 16-bit float precision instead of 32-bit") parser.add_argument( '--fp16_opt_level', type=str, default='O2', help= "For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']." "See details at https://nvidia.github.io/apex/amp.html") args = parser.parse_args() summary_writer = SummaryWriter(args.log_dir) device = torch.device( "cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") n_gpu = torch.cuda.device_count() logger.info("device: {} n_gpu: {}, 16-bits training: {}".format( device, n_gpu, args.fp16)) random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if n_gpu > 0: torch.cuda.manual_seed_all(args.seed) if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) # Prepare model config = Config.from_json_file(args.model_config) model = BertForPostTraining(config) if args.resume_checkpoint: model.load_state_dict(torch.load(args.checkpoint)) else: model.bert.load_state_dict(torch.load(args.checkpoint)) # Multi-GPU Setting if n_gpu > 1: model = nn.DataParallel(model) num_params = count_parameters(model) logger.info("Total Parameter: %d" % num_params) model.to(device) post_training_dataset = BertPostTrainingDataset(args.corpus, args.max_seq_length) num_train_optimization_steps = int( len(post_training_dataset) / args.train_batch_size) * args.num_train_epochs # Prepare optimizer param_optimizer = list(model.named_parameters()) no_decay = ['bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [{ 'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01 }, { 'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0 }] optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) scheduler = WarmupLinearSchedule( optimizer, warmup_steps=num_train_optimization_steps * 0.1, t_total=num_train_optimization_steps) if args.fp16: try: from apex import amp except ImportError: raise ImportError( "Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training." ) model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level) logger.info("***** Running training *****") logger.info(" Num orig examples = %d", len(post_training_dataset)) logger.info(" Batch size = %d", args.train_batch_size) logger.info(" Num steps = %d", num_train_optimization_steps) num_train_step = num_train_optimization_steps train_sampler = RandomSampler(post_training_dataset) train_dataloader = DataLoader(post_training_dataset, sampler=train_sampler, batch_size=args.train_batch_size, num_workers=16, pin_memory=True) model.train() global_step = 0 epoch = 0 for _ in trange(int(args.num_train_epochs), desc="Epoch"): iter_bar = tqdm( train_dataloader, desc="Train(XX Epoch) Step(XX/XX) (Mean loss=X.X) (loss=X.X)") tr_step, total_loss, mean_loss = 0, 0., 0. for step, batch in enumerate(iter_bar): # loss, mlm_loss, nsp_loss = model(batch) output = model(batch) loss, mlm_loss, nsp_loss = output[:3] if n_gpu > 1: loss = loss.mean() if args.fp16: with amp.scale_loss(loss, optimizer) as scaled_loss: scaled_loss.backward() torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm) else: loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm) optimizer.step() scheduler.step() optimizer.zero_grad() global_step += 1 tr_step += 1 total_loss += loss mean_loss = total_loss / tr_step iter_bar.set_description( "Train Step(%d / %d) (Mean loss=%5.5f) (loss=%5.5f)" % (global_step, num_train_step, mean_loss, loss.item())) if global_step % 100 == 0: summary_writer.add_scalar('Train/Total_Mean_Loss', mean_loss, global_step) summary_writer.add_scalar('Train/Total_Loss', loss.item(), global_step) summary_writer.add_scalar('Train/MLM_Loss', mlm_loss.mean().item(), global_step) summary_writer.add_scalar('Train/NSP_Loss', nsp_loss.mean().item(), global_step) logger.info("***** Saving file *****") if args.resume_checkpoint: model_checkpoint = "pt_bert_from_checkpoint_%d.bin" % (epoch) else: model_checkpoint = "pt_bert_%d.bin" % (epoch) logger.info(model_checkpoint) output_model_file = os.path.join(args.output_dir, model_checkpoint) if n_gpu > 1: torch.save(model.module.state_dict(), output_model_file) else: torch.save(model.state_dict(), output_model_file) epoch += 1
def main(): parser = argparse.ArgumentParser() # Required parameters parser.add_argument( "--output_dir", default='output', type=str, help= "The output directory where the model checkpoints and predictions will be written." ) parser.add_argument("--checkpoint", default='pretrain_ckpt/bert_small_ckpt.bin', type=str, help="checkpoint") parser.add_argument("--model_config", default='data/bert_small.json', type=str) parser.add_argument("--vocab", type=str) # Other parameters parser.add_argument("--train_file", default='data/KorQuAD_v1.0_train.json', type=str, help="SQuAD json for training. E.g., train-v1.1.json") parser.add_argument( "--max_seq_length", default=512, type=int, help= "The maximum total input sequence length after WordPiece tokenization. Sequences " "longer than this will be truncated, and sequences shorter than this will be padded." ) parser.add_argument( "--doc_stride", default=128, type=int, help= "When splitting up a long document into chunks, how much stride to take between chunks." ) parser.add_argument( "--max_query_length", default=96, type=int, help= "The maximum number of tokens for the question. Questions longer than this will " "be truncated to this length.") parser.add_argument("--train_batch_size", default=16, type=int, help="Total batch size for training.") parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument("--num_train_epochs", default=4.0, type=float, help="Total number of training epochs to perform.") parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") parser.add_argument("--adam_epsilon", default=1e-6, type=float, help="Epsilon for Adam optimizer.") parser.add_argument( "--warmup_proportion", default=0.1, type=float, help= "Proportion of training to perform linear learning rate warmup for. E.g., 0.1 = 10%% " "of training.") parser.add_argument( "--n_best_size", default=20, type=int, help= "The total number of n-best predictions to generate in the nbest_predictions.json " "output file.") parser.add_argument( "--max_answer_length", default=30, type=int, help= "The maximum length of an answer that can be generated. This is needed because the start " "and end predictions are not conditioned on one another.") parser.add_argument( "--verbose_logging", action='store_true', help= "If true, all of the warnings related to data processing will be printed. " "A number of warnings are expected for a normal SQuAD evaluation.") parser.add_argument("--no_cuda", action='store_true', help="Whether not to use CUDA when available") parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") parser.add_argument( '--fp16', action='store_true', help="Whether to use 16-bit float precision instead of 32-bit") parser.add_argument( '--fp16_opt_level', type=str, default='O2', help= "For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']." "See details at https://nvidia.github.io/apex/amp.html") parser.add_argument( '--null_score_diff_threshold', type=float, default=0.0, help= "If null_score - best_non_null is greater than the threshold predict null." ) parser.add_argument('--grad_noise', default='false', type=str) parser.add_argument('--gs_noise', default='false', type=str) parser.add_argument('--max_num_example', default=80000, type=int) args = parser.parse_args() args.grad_noise = args.grad_noise.lower() == 'true' args.gs_noise = args.gs_noise.lower() == 'true' device = torch.device( "cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") n_gpu = torch.cuda.device_count() logger.info("device: {} n_gpu: {}, 16-bits training: {}".format( device, n_gpu, args.fp16)) random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if n_gpu > 0: torch.cuda.manual_seed_all(args.seed) if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) ckpt_list_fn = os.path.join(args.output_dir, "ckpt_list.txt") if os.path.isfile(ckpt_list_fn): os.remove(ckpt_list_fn) tokenizer = BertTokenizer(args.vocab, max_len=args.max_seq_length, do_basic_tokenize=True) # Prepare model config = Config.from_json_file(args.model_config) model = QuestionAnswering(config) if 'bert' in args.checkpoint: model.bert.load_state_dict(torch.load(args.checkpoint)) logger.info("bert parameters are loaded from %s" % args.checkpoint) else: model.load_state_dict(torch.load(args.checkpoint)) logger.info("all parameter is loaded from %s" % args.checkpoint) num_params = count_parameters(model) logger.info("Total Parameter: %d" % num_params) model.to(device) train_examples = read_squad_examples(input_file=args.train_file, is_training=True, version_2_with_negative=False) num_train_examples = len(train_examples) cached_train_features_file = args.train_file + '_{0}_{1}_{2}'.format( str(args.max_seq_length), str(args.doc_stride), str(args.max_query_length)) is_dynamic = False if num_train_examples <= args.max_num_example: try: with open(cached_train_features_file, "rb") as reader: train_features = pickle.load(reader) except: train_features = convert_examples_to_features( examples=train_examples, tokenizer=tokenizer, max_seq_length=args.max_seq_length, doc_stride=args.doc_stride, max_query_length=args.max_query_length, is_training=True, is_dynamic=False) logger.info(" Saving train features into cached file %s", cached_train_features_file) with open(cached_train_features_file, "wb") as writer: pickle.dump(train_features, writer) del train_examples num_train_features = len(train_features) all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long) all_start_positions = torch.tensor( [f.start_position for f in train_features], dtype=torch.long) all_end_positions = torch.tensor( [f.end_position for f in train_features], dtype=torch.long) train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_start_positions, all_end_positions) train_sampler = RandomSampler(train_data) train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size) else: # For being able to get data dynamically (for large datasets) is_dynamic = True pkl_list = [] num_train_features = 0 if not os.path.isfile(cached_train_features_file + "_0"): batch = 10000 unique_id = 1000000000 for start, end in zip( range(0, len(train_examples), batch), range(batch, len(train_examples) + batch, batch)): file_path = cached_train_features_file + "_%d" % start unique_id = convert_examples_to_features( examples=train_examples[start:end], tokenizer=tokenizer, max_seq_length=args.max_seq_length, doc_stride=args.doc_stride, max_query_length=args.max_query_length, is_training=True, is_dynamic=True, file_path=file_path, init_unique_id=unique_id) pkl_list.append(file_path) num_train_features = unique_id - 1000000000 with open(cached_train_features_file + "_info", 'w') as fw: fw.write("%d\n" % num_train_features) for pkl in pkl_list: fw.write(pkl + '\n') else: with open(cached_train_features_file + "_info", 'r') as f: line_cnt = 0 for line in f: if line_cnt == 0: num_train_features = int(line.strip()) else: pkl_list.append(line.strip()) line_cnt += 1 del train_examples train_data = KorquadDataset(pkl_list, args.train_batch_size, tokenizer) num_train_optimization_steps = int( num_train_features / args.train_batch_size) * args.num_train_epochs # Prepare optimizer param_optimizer = list(model.named_parameters()) no_decay = ['bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [{ 'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01 }, { 'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0 }] optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) scheduler = WarmupLinearSchedule( optimizer, warmup_steps=num_train_optimization_steps * 0.1, t_total=num_train_optimization_steps) if args.fp16: try: from apex import amp except ImportError: raise ImportError( "Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training." ) model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level) logger.info("***** Running training *****") logger.info(" Num orig examples = %d", num_train_examples) logger.info(" Num split examples = %d", num_train_features) logger.info(" Batch size = %d", args.train_batch_size) logger.info(" Num steps = %d", num_train_optimization_steps) num_train_step = num_train_optimization_steps model.train() global_step = 0 epoch = 0 for _ in trange(int(args.num_train_epochs), desc="Epoch"): if is_dynamic: num_step = int(num_train_step / args.num_train_epochs) if num_step * args.train_batch_size != num_train_features: num_step += 1 iter_bar = tqdm( list(range(num_step)), desc="Train(XX Epoch) Step(XX/XX) (Mean loss=X.X) (loss=X.X)") else: iter_bar = tqdm( train_dataloader, desc="Train(XX Epoch) Step(XX/XX) (Mean loss=X.X) (loss=X.X)") tr_step, total_loss, mean_loss = 0, 0., 0. for step, batch in enumerate(iter_bar): if is_dynamic: input_ids, input_mask, segment_ids, start_positions, end_positions = train_data.next_batch( ) if n_gpu == 1: input_ids = input_ids.to(device) input_mask = input_mask.to(device) segment_ids = segment_ids.to(device) start_positions = start_positions.to(device) end_positions = end_positions.to(device) else: if n_gpu == 1: batch = tuple( t.to(device) for t in batch) # multi-gpu does scattering it-self input_ids, input_mask, segment_ids, start_positions, end_positions = batch if args.grad_noise: loss = model(input_ids, segment_ids, input_mask, start_positions, end_positions, cal_grad=True) else: loss = model(input_ids, segment_ids, input_mask, start_positions, end_positions) if n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu. if args.fp16: with amp.scale_loss(loss, optimizer) as scaled_loss: scaled_loss.backward() torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm) else: loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm) if args.grad_noise or args.gs_noise: if args.grad_noise: adv_grad = model.bert.embedding_output.grad adv_loss = model(input_ids, segment_ids, input_mask, start_positions, end_positions, adv_grad, cal_grad=False, is_gs_noise=False) else: adv_loss = model(input_ids, segment_ids, input_mask, start_positions, end_positions, adv_grad=None, cal_grad=False, is_gs_noise=True) if n_gpu > 1: adv_loss = adv_loss.mean( ) # mean() to average on multi-gpu. if args.fp16: with amp.scale_loss(adv_loss, optimizer) as scaled_loss: scaled_loss.backward() torch.nn.utils.clip_grad_norm_( amp.master_params(optimizer), args.max_grad_norm) else: adv_loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm) scheduler.step() optimizer.step() optimizer.zero_grad() global_step += 1 tr_step += 1 total_loss += loss.item() mean_loss = total_loss / tr_step iter_bar.set_description( "Train Step(%d / %d) (Mean loss=%5.5f) (loss=%5.5f)" % (global_step, num_train_step, mean_loss, loss.item())) logger.info("** ** * Saving file * ** **") model_checkpoint = "korquad_%d.bin" % (epoch) logger.info(model_checkpoint) output_model_file = os.path.join(args.output_dir, model_checkpoint) if n_gpu > 1: torch.save(model.module.state_dict(), output_model_file) else: torch.save(model.state_dict(), output_model_file) with open(os.path.join(args.output_dir, "ckpt_list.txt"), 'a') as fw: fw.write(model_checkpoint + '\n') epoch += 1
def main(input, output): max_seq_length = 512 doc_stride = 64 max_query_length = 64 batch_size = 16 n_best_size = 20 max_answer_length = 30 seed = 42 fp16 = False device = torch.device("cpu") # device = torch.device("cuda" if torch.cuda.is_available() else "cpu") n_gpu = torch.cuda.device_count() logger.info("device: {} n_gpu: {}".format(device, n_gpu)) random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) if n_gpu > 0: torch.cuda.manual_seed_all(seed) tokenizer = BertTokenizer(vocab_file=VOCAB_PATH, max_len=max_seq_length, do_basic_tokenize=True) eval_examples = read_squad_examples(input_file=input, is_training=False, version_2_with_negative=False) eval_features = convert_examples_to_features( examples=eval_examples, tokenizer=tokenizer, max_seq_length=max_seq_length, doc_stride=doc_stride, max_query_length=max_query_length, is_training=False) # Prepare model config = Config.from_json_file(CONFIG_PATH) model = QuestionAnswering(config) if fp16 is True: model.half() model.load_state_dict(torch.load(CHK_PATH, map_location=device)) model.to(device) logger.info("***** Running training *****") logger.info(" Num orig examples = %d", len(eval_examples)) logger.info(" Num split examples = %d", len(eval_features)) logger.info(" Batch size = %d", batch_size) all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long) all_example_index = torch.arange(all_input_ids.size(0), dtype=torch.long) eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_example_index) eval_sampler = SequentialSampler(eval_data) eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=batch_size) model.eval() all_results = [] logger.info("Start evaluating") for input_ids, input_mask, segment_ids, example_indices in tqdm( eval_dataloader, desc="Evaluating"): input_ids = input_ids.to(device) input_mask = input_mask.to(device) segment_ids = segment_ids.to(device) with torch.no_grad(): batch_start_logits, batch_end_logits = model( input_ids, segment_ids, input_mask) for i, example_index in enumerate(example_indices): start_logits = batch_start_logits[i].detach().cpu().tolist() end_logits = batch_end_logits[i].detach().cpu().tolist() eval_feature = eval_features[example_index.item()] unique_id = int(eval_feature.unique_id) all_results.append( RawResult(unique_id=unique_id, start_logits=start_logits, end_logits=end_logits)) output_nbest_file = os.path.join("nbest_predictions.json") write_predictions(eval_examples, eval_features, all_results, n_best_size, max_answer_length, False, output, output_nbest_file, None, False, False, 0.0)
def main(): parser = argparse.ArgumentParser() # Parameters parser.add_argument( "--output_dir", default='debug', type=str, help= "The output directory where the model checkpoints and predictions will be written." ) parser.add_argument("--checkpoint", default='output/korquad_3.bin', type=str, help="fine-tuned model checkpoint") parser.add_argument("--config_file", default='data/large_config.json', type=str, help="model configuration file") parser.add_argument("--vocab_file", default='data/large_v1_32k_vocab.txt', type=str, help="tokenizer vocab file") parser.add_argument("--check_list", default=False, type=bool) parser.add_argument("--eda_type", default="no_eda", type=str, help="sr, ri , rd, rs") parser.add_argument( "--predict_file", default='data/korquad/KorQuAD_v1.0_dev.json', type=str, help="SQuAD json for predictions. E.g., dev-v1.1.json or test-v1.1.json" ) parser.add_argument( "--max_seq_length", default=512, type=int, help= "The maximum total input sequence length after WordPiece tokenization. Sequences " "longer than this will be truncated, and sequences shorter than this will be padded." ) parser.add_argument( "--doc_stride", default=64, type=int, help= "When splitting up a long document into chunks, how much stride to take between chunks." ) parser.add_argument( "--max_query_length", default=64, type=int, help= "The maximum number of tokens for the question. Questions longer than this will " "be truncated to this length.") parser.add_argument( "--max_answer_length", default=30, type=int, help= "The maximum length of an answer that can be generated. This is needed because the start " "and end predictions are not conditioned on one another.") parser.add_argument("--batch_size", default=16, type=int, help="Batch size per GPU/CPU for evaluation.") parser.add_argument( "--n_best_size", default=20, type=int, help= "The total number of n-best predictions to generate in the nbest_predictions." "json output file.") parser.add_argument( "--verbose_logging", action='store_true', help= "If true, all of the warnings related to data processing will be printed. " "A number of warnings are expected for a normal SQuAD evaluation.") parser.add_argument("--just_eval", default=False, type=bool) parser.add_argument("--no_cuda", action='store_true', help="Whether not to use CUDA when available") parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") parser.add_argument( '--fp16', action='store_true', help= "Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit" ) args = parser.parse_args() # Setup CUDA, GPU & distributed training device = torch.device( "cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") args.n_gpu = torch.cuda.device_count() args.device = device # Setup logging logging.basicConfig( format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO) logger.info("device: %s, n_gpu: %s, 16-bits training: %s", device, args.n_gpu, args.fp16) # Set seed set_seed(args) tokenizer = BertTokenizer(vocab_file=args.vocab_file, do_basic_tokenize=True, max_len=args.max_seq_length) config = Config.from_json_file(args.config_file) model = QuestionAnswering(config) if args.check_list: model_list = os.listdir(args.checkpoint) model_list.sort() result_dict = defaultdict(list) eda_list = ["st_rs", "st_rd", "rs", "rd", "sr", "ri", "no_eda"] for m in model_list: model.load_state_dict(torch.load(args.checkpoint + '/' + m)) num_params = count_parameters(model) logger.info("Total Parameter: %d" % num_params) if args.fp16: model.half() model.to(args.device) logger.info("Evaluation parameters %s", args) for eda in eda_list: if eda in m: args.eda_type = eda break # Evaluate examples, features = load_and_cache_examples(args, tokenizer) r = evaluate(args, model, examples, features) em = r["exact_match"] f1 = r["f1"] result_dict[args.eda_type].append([em, f1, m]) print(result_dict) else: model.load_state_dict(torch.load(args.checkpoint)) num_params = count_parameters(model) logger.info("Total Parameter: %d" % num_params) if args.fp16: model.half() model.to(args.device) logger.info("Evaluation parameters %s", args) # Evaluate examples, features = load_and_cache_examples(args, tokenizer) evaluate(args, model, examples, features)