def optimizer_fn(param_group, max_grad_norm=None): group0 = dict(params=[], weight_decay_rate=args.weight_decay, names=[]) group1 = dict(params=[], weight_decay_rate=0.00, names=[]) for (n, p) in param_group: if not any(nd in n for nd in no_decay): group0['params'].append(p) group0['names'].append(n) else: group1['params'].append(p) group1['names'].append(n) optimizer_grouped_parameters = [group0, group1] optimizer = BertAdam( optimizer_grouped_parameters, lr=args.learning_rate, b1=args.adam_beta1, b2=args.adam_beta2, v1=args.qhadam_v1, v2=args.qhadam_v2, lr_ends=args.lr_schedule_ends, warmup=args.warmup_proportion if args.warmup_proportion < 1 else args.warmup_proportion / training_steps, t_total=training_steps, schedule=args.lr_schedule, max_grad_norm=args.max_grad_norm if max_grad_norm is None else max_grad_norm, global_grad_norm=args.global_grad_norm, init_spec=init_spec, weight_decay_rate=args.weight_decay) return optimizer
'weight_decay_rate': args.weight_decay_rate }, { 'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay_rate': 0.0 }] num_train_steps = int( len(train_iter.dataset) / args.batch_size * device_num) * args.num_epochs num_train_steps = num_train_steps if args.t_total else -1 optimizer = BertAdam(params=optimizer_grouped_parameters, lr=args.bert_lr, warmup=args.warmup, t_total=num_train_steps) for epoch in range(1, args.num_epochs + 1): logger.info( "==========epoch {} fine tune start==========".format(epoch)) logger.info('train examples {}'.format(len(train_iter.dataset))) logger.info('train batch size {}'.format(args.batch_size)) logger.info('train lr {}'.format(optimizer.get_lr()[0])) writer.add_scalar('lr', optimizer.get_lr()[0], epoch) train(model, train_iter, optimizer, epoch) logger.info( "==========epoch {} fine tune end==========".format(epoch)) logger.info( "==========epoch {} eval start==========".format(epoch)) evaluate(model, eval_iter, epoch)
def main(): parser = argparse.ArgumentParser() ## Required parameters parser.add_argument( "--data_dir", default=None, type=str, required=True, help= "The input data dir. Should contain the .tsv files (or other data files) for the task." ) parser.add_argument( "--bert_model", default=None, type=str, required=True, help="Bert pre-trained model selected in the list: bert-base-uncased, " "bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, " "bert-base-multilingual-cased, bert-base-chinese.") parser.add_argument("--task_name", default=None, type=str, required=True, help="The name of the task to train.") parser.add_argument( "--output_dir", default=None, type=str, required=True, help= "The output directory where the model predictions and checkpoints will be written." ) parser.add_argument("--negative_weight", default=1., type=float) parser.add_argument("--neutral_words_file", default='data/identity.csv') # if true, use test data instead of val data parser.add_argument("--test", action='store_true') # Explanation specific arguments below # whether run explanation algorithms parser.add_argument("--explain", action='store_true', help='if true, explain test set predictions') parser.add_argument("--debug", action='store_true') # which algorithm to run parser.add_argument("--algo", choices=['soc']) # the output filename without postfix parser.add_argument("--output_filename", default='temp.tmp') # see utils/config.py parser.add_argument("--use_padding_variant", action='store_true') parser.add_argument("--mask_outside_nb", action='store_true') parser.add_argument("--nb_range", type=int) parser.add_argument("--sample_n", type=int) # whether use explanation regularization parser.add_argument("--reg_explanations", action='store_true') parser.add_argument("--reg_strength", type=float) parser.add_argument("--reg_mse", action='store_true') # whether discard other neutral words during regularization. default: False parser.add_argument("--discard_other_nw", action='store_false', dest='keep_other_nw') # whether remove neutral words when loading datasets parser.add_argument("--remove_nw", action='store_true') # if true, generate hierarchical explanations instead of word level outputs. # Only useful when the --explain flag is also added. parser.add_argument("--hiex", action='store_true') parser.add_argument("--hiex_tree_height", default=5, type=int) # whether add the sentence itself to the sample set in SOC parser.add_argument("--hiex_add_itself", action='store_true') # the directory where the lm is stored parser.add_argument("--lm_dir", default='runs/lm') # if configured, only generate explanations for instances with given line numbers parser.add_argument("--hiex_idxs", default=None) # if true, use absolute values of explanations for hierarchical clustering parser.add_argument("--hiex_abs", action='store_true') # if either of the two is true, only generate explanations for positive / negative instances parser.add_argument("--only_positive", action='store_true') parser.add_argument("--only_negative", action='store_true') # stop after generating x explanation parser.add_argument("--stop", default=100000000, type=int) # early stopping with decreasing learning rate. 0: direct exit when validation F1 decreases parser.add_argument("--early_stop", default=5, type=int) # other external arguments originally here in pytorch_transformers parser.add_argument( "--cache_dir", default="", type=str, help= "Where do you want to store the pre-trained models downloaded from s3") parser.add_argument( "--max_seq_length", default=128, type=int, help= "The maximum total input sequence length after WordPiece tokenization. \n" "Sequences longer than this will be truncated, and sequences shorter \n" "than this will be padded.") parser.add_argument("--do_train", action='store_true', help="Whether to run training.") parser.add_argument("--do_eval", action='store_true', help="Whether to run eval on the dev set.") parser.add_argument( "--do_lower_case", action='store_true', help="Set this flag if you are using an uncased model.") parser.add_argument("--train_batch_size", default=32, type=int, help="Total batch size for training.") parser.add_argument("--eval_batch_size", default=32, type=int, help="Total batch size for eval.") parser.add_argument("--validate_steps", default=200, type=int, help="validate once for how many steps") parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument("--num_train_epochs", default=3.0, type=float, help="Total number of training epochs to perform.") parser.add_argument( "--warmup_proportion", default=0.1, type=float, help= "Proportion of training to perform linear learning rate warmup for. " "E.g., 0.1 = 10%% of training.") parser.add_argument("--no_cuda", action='store_true', help="Whether not to use CUDA when available") parser.add_argument("--local_rank", type=int, default=-1, help="local_rank for distributed training on gpus") parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") parser.add_argument( '--gradient_accumulation_steps', type=int, default=1, help= "Number of updates steps to accumulate before performing a backward/update pass." ) parser.add_argument( '--fp16', action='store_true', help="Whether to use 16-bit float precision instead of 32-bit") parser.add_argument( '--loss_scale', type=float, default=0, help= "Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n" "0 (default value): dynamic loss scaling.\n" "Positive power of 2: static loss scaling value.\n") parser.add_argument('--server_ip', type=str, default='', help="Can be used for distant debugging.") parser.add_argument('--server_port', type=str, default='', help="Can be used for distant debugging.") args = parser.parse_args() combine_args(configs, args) args = configs if args.server_ip and args.server_port: # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script import ptvsd print("Waiting for debugger attach") ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True) ptvsd.wait_for_attach() processors = { 'gab': GabProcessor, 'ws': WSProcessor, 'nyt': NytProcessor, 'MT': MTProcessor, #'multi-label': multilabel_Processor, } output_modes = { 'gab': 'classification', 'ws': 'classification', 'nyt': 'classification' } if args.local_rank == -1 or args.no_cuda: device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") n_gpu = torch.cuda.device_count() else: torch.cuda.set_device(args.local_rank) device = torch.device("cuda", args.local_rank) n_gpu = 1 # Initializes the distributed backend which will take care of sychronizing nodes/GPUs torch.distributed.init_process_group(backend='nccl') logging.basicConfig( format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN) logger.info( "device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}". format(device, n_gpu, bool(args.local_rank != -1), args.fp16)) if args.gradient_accumulation_steps < 1: raise ValueError( "Invalid gradient_accumulation_steps parameter: {}, should be >= 1" .format(args.gradient_accumulation_steps)) args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if n_gpu > 0: torch.cuda.manual_seed_all(args.seed) if not args.do_train and not args.do_eval: raise ValueError( "At least one of `do_train` or `do_eval` must be True.") #if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train: # raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir)) if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) # save configs f = open(os.path.join(args.output_dir, 'args.json'), 'w') json.dump(args.__dict__, f, indent=4) f.close() task_name = args.task_name.lower() if task_name not in processors: raise ValueError("Task not found: %s" % (task_name)) tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case) processor = processors[task_name](configs, tokenizer=tokenizer) output_mode = output_modes[task_name] label_list = processor.get_labels() num_labels = len(label_list) train_examples = None num_train_optimization_steps = None if args.do_train: train_examples = processor.get_train_examples(args.data_dir) num_train_optimization_steps = int( len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps) * args.num_train_epochs if args.local_rank != -1: num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size( ) # Prepare model cache_dir = args.cache_dir if args.cache_dir else os.path.join( str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed_{}'.format( args.local_rank)) if args.do_train: model = BertForSequenceClassification.from_pretrained( args.bert_model, cache_dir=cache_dir, num_labels=num_labels) else: model = BertForSequenceClassification.from_pretrained( args.output_dir, num_labels=num_labels) model.to(device) if args.fp16: model.half() if args.local_rank != -1: try: from apex.parallel import DistributedDataParallel as DDP except ImportError: raise ImportError( "Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training." ) model = DDP(model) # elif n_gpu > 1: # model = torch.nn.DataParallel(model) # Prepare optimizer param_optimizer = list(model.named_parameters()) no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [{ 'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01 }, { 'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0 }] if args.fp16: try: from apex.optimizers import FP16_Optimizer from apex.optimizers import FusedAdam except ImportError: raise ImportError( "Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training." ) optimizer = FusedAdam(optimizer_grouped_parameters, lr=args.learning_rate, bias_correction=False, max_grad_norm=1.0) if args.loss_scale == 0: optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True) else: optimizer = FP16_Optimizer(optimizer, static_loss_scale=args.loss_scale) warmup_linear = WarmupLinearSchedule( warmup=args.warmup_proportion, t_total=num_train_optimization_steps) else: if args.do_train: optimizer = BertAdam(optimizer_grouped_parameters, lr=args.learning_rate, warmup=args.warmup_proportion, t_total=num_train_optimization_steps) global_step = 0 nb_tr_steps = 0 tr_loss, tr_reg_loss = 0, 0 tr_reg_cnt = 0 epoch = -1 val_best_f1 = -1 val_best_loss = 1e10 early_stop_countdown = args.early_stop if args.reg_explanations: train_lm_dataloder = processor.get_dataloader('train', configs.train_batch_size) dev_lm_dataloader = processor.get_dataloader('dev', configs.train_batch_size) explainer = SamplingAndOcclusionExplain( model, configs, tokenizer, device=device, vocab=tokenizer.vocab, train_dataloader=train_lm_dataloder, dev_dataloader=dev_lm_dataloader, lm_dir=args.lm_dir, output_path=os.path.join(configs.output_dir, configs.output_filename), ) else: explainer = None if args.do_train: epoch = 0 train_features = convert_examples_to_features(train_examples, label_list, args.max_seq_length, tokenizer, output_mode, configs) logger.info("***** Running training *****") logger.info(" Num examples = %d", len(train_examples)) logger.info(" Batch size = %d", args.train_batch_size) logger.info(" Num steps = %d", num_train_optimization_steps) all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long) if output_mode == "classification": all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long) elif output_mode == "regression": all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.float) train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) if args.local_rank == -1: train_sampler = RandomSampler(train_data) else: train_sampler = DistributedSampler(train_data) train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size) class_weight = torch.FloatTensor([args.negative_weight, 1]).to(device) model.train() for _ in trange(int(args.num_train_epochs), desc="Epoch"): tr_loss = 0 nb_tr_examples, nb_tr_steps = 0, 0 for step, batch in enumerate( tqdm(train_dataloader, desc="Iteration")): batch = tuple(t.to(device) for t in batch) input_ids, input_mask, segment_ids, label_ids = batch # define a new function to compute loss values for both output_modes logits = model(input_ids, segment_ids, input_mask, labels=None) if output_mode == "classification": loss_fct = CrossEntropyLoss(class_weight) loss = loss_fct(logits.view(-1, num_labels), label_ids.view(-1)) elif output_mode == "regression": loss_fct = MSELoss() loss = loss_fct(logits.view(-1), label_ids.view(-1)) if n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu. if args.gradient_accumulation_steps > 1: loss = loss / args.gradient_accumulation_steps tr_loss += loss.item() if args.fp16: optimizer.backward(loss) else: loss.backward() # regularize explanations # NOTE: backward performed inside this function to prevent OOM if args.reg_explanations: reg_loss, reg_cnt = explainer.compute_explanation_loss( input_ids, input_mask, segment_ids, label_ids, do_backprop=True) tr_reg_loss += reg_loss # float tr_reg_cnt += reg_cnt nb_tr_examples += input_ids.size(0) nb_tr_steps += 1 if (step + 1) % args.gradient_accumulation_steps == 0: if args.fp16: # modify learning rate with special warm up BERT uses # if args.fp16 is False, BertAdam is used that handles this automatically lr_this_step = args.learning_rate * warmup_linear.get_lr( global_step / num_train_optimization_steps, args.warmup_proportion) for param_group in optimizer.param_groups: param_group['lr'] = lr_this_step optimizer.step() optimizer.zero_grad() global_step += 1 if global_step % args.validate_steps == 0: val_result = validate(args, model, processor, tokenizer, output_mode, label_list, device, num_labels, task_name, tr_loss, global_step, epoch, explainer) val_acc, val_f1 = val_result['acc'], val_result['f1'] if val_f1 > val_best_f1: val_best_f1 = val_f1 if args.local_rank == -1 or torch.distributed.get_rank( ) == 0: save_model(args, model, tokenizer, num_labels) else: # halve the learning rate for param_group in optimizer.param_groups: param_group['lr'] *= 0.5 early_stop_countdown -= 1 logger.info( "Reducing learning rate... Early stop countdown %d" % early_stop_countdown) if early_stop_countdown < 0: break if early_stop_countdown < 0: break epoch += 1 # training finish ############################ # if args.do_eval and (args.local_rank == -1 or torch.distributed.get_rank() == 0): # if not args.explain: # args.test = True # validate(args, model, processor, tokenizer, output_mode, label_list, device, num_labels, # task_name, tr_loss, global_step=0, epoch=-1, explainer=explainer) # else: # args.test = True # explain(args, model, processor, tokenizer, output_mode, label_list, device) if not args.explain: args.test = True print('--Test_args.test: %s' % str(args.test)) #Test_args.test: True validate(args, model, processor, tokenizer, output_mode, label_list, device, num_labels, task_name, tr_loss, global_step=888, epoch=-1, explainer=explainer) args.test = False else: print('--Test_args.test: %s' % str(args.test)) # Test_args.test: True args.test = True explain(args, model, processor, tokenizer, output_mode, label_list, device) args.test = False
def main(): parser = ArgumentParser() parser.add_argument('--pregenerated_data', type=Path, required=True) parser.add_argument('--output_dir', type=Path, required=True) parser.add_argument( "--bert_model", type=str, required=True, help="Bert pre-trained model selected in the list: bert-base-uncased, " "bert-large-uncased, bert-base-cased, bert-base-multilingual, bert-base-chinese." ) parser.add_argument("--do_lower_case", action="store_true") parser.add_argument( "--reduce_memory", action="store_true", help= "Store training data as on-disc memmaps to massively reduce memory usage" ) parser.add_argument("--epochs", type=int, default=3, help="Number of epochs to train for") parser.add_argument("--local_rank", type=int, default=-1, help="local_rank for distributed training on gpus") parser.add_argument("--no_cuda", action='store_true', help="Whether not to use CUDA when available") parser.add_argument( '--gradient_accumulation_steps', type=int, default=1, help= "Number of updates steps to accumulate before performing a backward/update pass." ) parser.add_argument("--train_batch_size", default=32, type=int, help="Total batch size for training.") parser.add_argument( '--fp16', action='store_true', help="Whether to use 16-bit float precision instead of 32-bit") parser.add_argument( '--loss_scale', type=float, default=0, help= "Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n" "0 (default value): dynamic loss scaling.\n" "Positive power of 2: static loss scaling value.\n") parser.add_argument( "--warmup_proportion", default=0.1, type=float, help= "Proportion of training to perform linear learning rate warmup for. " "E.g., 0.1 = 10%% of training.") parser.add_argument("--learning_rate", default=3e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") parser.add_argument('--log_every', type=int, default=100, help="Log every X batch") parser.add_argument("--mlm_only", action='store_true', help="Only use MLM objective") args = parser.parse_args() assert args.pregenerated_data.is_dir(), \ "--pregenerated_data should point to the folder of files made by pregenerate_training_data.py!" if args.output_dir.is_dir() and list(args.output_dir.iterdir()): print( f"Output directory ({args.output_dir}) already exists and is not empty!" ) args.output_dir.mkdir(parents=True, exist_ok=True) logger = util.get_logger(f'{args.output_dir}/exp.txt') for key, value in vars(args).items(): logger.info('command line argument: %s - %r', key, value) samples_per_epoch = [] for i in range(args.epochs): epoch_file = args.pregenerated_data / f"epoch_{i}.json" metrics_file = args.pregenerated_data / f"epoch_{i}_metrics.json" if epoch_file.is_file() and metrics_file.is_file(): metrics = json.loads(metrics_file.read_text()) samples_per_epoch.append(metrics['num_training_examples']) else: if i == 0: exit("No training data was found!") print( f"Warning! There are fewer epochs of pregenerated data ({i}) than training epochs ({args.epochs})." ) print( "This script will loop over the available data, but training diversity may be negatively impacted." ) num_data_epochs = i break else: num_data_epochs = args.epochs if args.local_rank == -1 or args.no_cuda: device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") n_gpu = torch.cuda.device_count() else: torch.cuda.set_device(args.local_rank) device = torch.device("cuda", args.local_rank) n_gpu = 1 # Initializes the distributed backend which will take care of sychronizing nodes/GPUs torch.distributed.init_process_group(backend='nccl') logger.info( "device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}". format(device, n_gpu, bool(args.local_rank != -1), args.fp16)) if args.gradient_accumulation_steps < 1: raise ValueError( "Invalid gradient_accumulation_steps parameter: {}, should be >= 1" .format(args.gradient_accumulation_steps)) args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if n_gpu > 0: torch.cuda.manual_seed_all(args.seed) tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case) total_train_examples = 0 for i in range(args.epochs): # The modulo takes into account the fact that we may loop over limited epochs of data total_train_examples += samples_per_epoch[i % len(samples_per_epoch)] num_train_optimization_steps = int(total_train_examples / args.train_batch_size / args.gradient_accumulation_steps) if args.local_rank != -1: num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size( ) # Prepare model if args.mlm_only: model = BertForMaskedLM.from_pretrained(args.bert_model) else: model = BertForPreTraining.from_pretrained(args.bert_model) if args.fp16: model.half() model.to(device) if args.local_rank != -1: try: from apex.parallel import DistributedDataParallel as DDP except ImportError: raise ImportError( "Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training." ) model = DDP(model) elif n_gpu > 1: model = torch.nn.DataParallel(model) # Prepare optimizer param_optimizer = list(model.named_parameters()) if args.mlm_only: param_optimizer = [ x for x in param_optimizer if 'bert.pooler' not in x[0] ] no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [{ 'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01 }, { 'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0 }] if args.fp16: try: from apex.optimizers import FP16_Optimizer from apex.optimizers import FusedAdam except ImportError: raise ImportError( "Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training." ) optimizer = FusedAdam(optimizer_grouped_parameters, lr=args.learning_rate, bias_correction=False, max_grad_norm=1.0) if args.loss_scale == 0: optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True) else: optimizer = FP16_Optimizer(optimizer, static_loss_scale=args.loss_scale) else: optimizer = BertAdam(optimizer_grouped_parameters, lr=args.learning_rate, warmup=args.warmup_proportion, t_total=num_train_optimization_steps) global_step = 0 logger.info("***** Running training *****") logger.info(f" Num examples = {total_train_examples}") logger.info(" Batch size = %d", args.train_batch_size) logger.info(" Num steps = %d", num_train_optimization_steps) model.train() for epoch in range(args.epochs): epoch_dataset = PregeneratedDataset( logger=logger, epoch=epoch, training_path=args.pregenerated_data, tokenizer=tokenizer, num_data_epochs=num_data_epochs, mlm_only=args.mlm_only) if args.local_rank == -1: train_sampler = RandomSampler(epoch_dataset) else: train_sampler = DistributedSampler(epoch_dataset) train_dataloader = DataLoader(epoch_dataset, sampler=train_sampler, batch_size=args.train_batch_size) tr_loss = 0 nb_tr_examples, nb_tr_steps = 0, 0 losses = [] with tqdm(total=len(train_dataloader), desc=f"Epoch {epoch}") as pbar: for step, batch in enumerate(train_dataloader): batch = tuple(t.to(device) for t in batch) if args.mlm_only: input_ids, input_mask, segment_ids, lm_label_ids = batch loss = model(input_ids, segment_ids, input_mask, lm_label_ids) else: input_ids, input_mask, segment_ids, lm_label_ids, is_next = batch loss = model(input_ids, segment_ids, input_mask, lm_label_ids, is_next) if n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu. if args.gradient_accumulation_steps > 1: loss = loss / args.gradient_accumulation_steps if args.fp16: optimizer.backward(loss) else: loss.backward() tr_loss += loss.item() nb_tr_examples += input_ids.size(0) nb_tr_steps += 1 pbar.update(1) mean_loss = tr_loss * args.gradient_accumulation_steps / nb_tr_steps pbar.set_postfix_str(f"Loss: {mean_loss:.5f}") losses.append(loss.item()) if step % args.log_every == 0: logger.info( f"loss at ep {epoch} batch {step}/{len(train_dataloader)} is {np.mean(losses):.5f}" ) losses = [] if (step + 1) % args.gradient_accumulation_steps == 0: if args.fp16: # modify learning rate with special warm up BERT uses # if args.fp16 is False, BertAdam is used that handles this automatically lr_this_step = args.learning_rate * warmup_linear( global_step / num_train_optimization_steps, args.warmup_proportion) for param_group in optimizer.param_groups: param_group['lr'] = lr_this_step optimizer.step() optimizer.zero_grad() global_step += 1 # Save a trained model logger.info("** ** * Saving fine-tuned model ** ** * ") model_to_save = model.module if hasattr( model, 'module') else model # Only save the model it-self output_model_file = args.output_dir / f"epoch{epoch}_pytorch_model.bin" torch.save(model_to_save.state_dict(), str(output_model_file))
def __init__(self, opt, state_dict=None, num_train_step=-1): self.config = opt self.updates = state_dict[ 'updates'] if state_dict and 'updates' in state_dict else 0 self.train_loss = AverageMeter() self.network = SANBertNetwork(opt) # pdb.set_trace() if state_dict: new_state = set(self.network.state_dict().keys()) # change to a safer approach old_keys = [k for k in state_dict['state'].keys()] for k in old_keys: if k not in new_state: print('deleting state:', k) del state_dict['state'][k] for k, v in list(self.network.state_dict().items()): if k not in state_dict['state']: print('adding missing state:', k) state_dict['state'][k] = v # pdb.set_trace() self.network.load_state_dict(state_dict['state']) self.mnetwork = nn.DataParallel( self.network) if opt['multi_gpu_on'] else self.network self.total_param = sum([ p.nelement() for p in self.network.parameters() if p.requires_grad ]) no_decay = [ 'bias', 'gamma', 'beta', 'LayerNorm.bias', 'LayerNorm.weight' ] optimizer_parameters = [{ 'params': [ p for n, p in self.network.named_parameters() if n not in no_decay ], 'weight_decay_rate': 0.01 }, { 'params': [p for n, p in self.network.named_parameters() if n in no_decay], 'weight_decay_rate': 0.0 }] # note that adamax are modified based on the BERT code if opt['optimizer'] == 'sgd': self.optimizer = optim.SGD(optimizer_parameters, opt['learning_rate'], weight_decay=opt['weight_decay']) elif opt['optimizer'] == 'adamax': self.optimizer = Adamax(optimizer_parameters, opt['learning_rate'], warmup=opt['warmup'], t_total=num_train_step, max_grad_norm=opt['grad_clipping'], schedule=opt['warmup_schedule']) if opt.get('have_lr_scheduler', False): opt['have_lr_scheduler'] = False elif opt['optimizer'] == 'adadelta': self.optimizer = optim.Adadelta(optimizer_parameters, opt['learning_rate'], rho=0.95) elif opt['optimizer'] == 'adam': self.optimizer = Adam(optimizer_parameters, lr=opt['learning_rate'], warmup=opt['warmup'], t_total=num_train_step, max_grad_norm=opt['grad_clipping'], schedule=opt['warmup_schedule']) if opt.get('have_lr_scheduler', False): opt['have_lr_scheduler'] = False else: raise RuntimeError('Unsupported optimizer: %s' % opt['optimizer']) if state_dict and 'optimizer' in state_dict: self.optimizer.load_state_dict(state_dict['optimizer']) if opt.get('have_lr_scheduler', False): if opt.get('scheduler_type', 'rop') == 'rop': self.scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=opt['lr_gamma'], patience=3) elif opt.get('scheduler_type', 'rop') == 'exp': self.scheduler = ExponentialLR(self.optimizer, gamma=opt.get('lr_gamma', 0.95)) else: milestones = [ int(step) for step in opt.get('multi_step_lr', '10,20,30').split(',') ] self.scheduler = MultiStepLR(self.optimizer, milestones=milestones, gamma=opt.get('lr_gamma')) else: self.scheduler = None self.ema = None if opt['ema_opt'] > 0: self.ema = EMA(self.config['ema_gamma'], self.network) self.para_swapped = False
class MTDNNModel(object): def __init__(self, opt, state_dict=None, num_train_step=-1): self.config = opt self.updates = state_dict[ 'updates'] if state_dict and 'updates' in state_dict else 0 self.train_loss = AverageMeter() self.network = SANBertNetwork(opt) # pdb.set_trace() if state_dict: new_state = set(self.network.state_dict().keys()) # change to a safer approach old_keys = [k for k in state_dict['state'].keys()] for k in old_keys: if k not in new_state: print('deleting state:', k) del state_dict['state'][k] for k, v in list(self.network.state_dict().items()): if k not in state_dict['state']: print('adding missing state:', k) state_dict['state'][k] = v # pdb.set_trace() self.network.load_state_dict(state_dict['state']) self.mnetwork = nn.DataParallel( self.network) if opt['multi_gpu_on'] else self.network self.total_param = sum([ p.nelement() for p in self.network.parameters() if p.requires_grad ]) no_decay = [ 'bias', 'gamma', 'beta', 'LayerNorm.bias', 'LayerNorm.weight' ] optimizer_parameters = [{ 'params': [ p for n, p in self.network.named_parameters() if n not in no_decay ], 'weight_decay_rate': 0.01 }, { 'params': [p for n, p in self.network.named_parameters() if n in no_decay], 'weight_decay_rate': 0.0 }] # note that adamax are modified based on the BERT code if opt['optimizer'] == 'sgd': self.optimizer = optim.SGD(optimizer_parameters, opt['learning_rate'], weight_decay=opt['weight_decay']) elif opt['optimizer'] == 'adamax': self.optimizer = Adamax(optimizer_parameters, opt['learning_rate'], warmup=opt['warmup'], t_total=num_train_step, max_grad_norm=opt['grad_clipping'], schedule=opt['warmup_schedule']) if opt.get('have_lr_scheduler', False): opt['have_lr_scheduler'] = False elif opt['optimizer'] == 'adadelta': self.optimizer = optim.Adadelta(optimizer_parameters, opt['learning_rate'], rho=0.95) elif opt['optimizer'] == 'adam': self.optimizer = Adam(optimizer_parameters, lr=opt['learning_rate'], warmup=opt['warmup'], t_total=num_train_step, max_grad_norm=opt['grad_clipping'], schedule=opt['warmup_schedule']) if opt.get('have_lr_scheduler', False): opt['have_lr_scheduler'] = False else: raise RuntimeError('Unsupported optimizer: %s' % opt['optimizer']) if state_dict and 'optimizer' in state_dict: self.optimizer.load_state_dict(state_dict['optimizer']) if opt.get('have_lr_scheduler', False): if opt.get('scheduler_type', 'rop') == 'rop': self.scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=opt['lr_gamma'], patience=3) elif opt.get('scheduler_type', 'rop') == 'exp': self.scheduler = ExponentialLR(self.optimizer, gamma=opt.get('lr_gamma', 0.95)) else: milestones = [ int(step) for step in opt.get('multi_step_lr', '10,20,30').split(',') ] self.scheduler = MultiStepLR(self.optimizer, milestones=milestones, gamma=opt.get('lr_gamma')) else: self.scheduler = None self.ema = None if opt['ema_opt'] > 0: self.ema = EMA(self.config['ema_gamma'], self.network) self.para_swapped = False def setup_ema(self): if self.config['ema_opt']: self.ema.setup() def update_ema(self): if self.config['ema_opt']: self.ema.update() def eval(self): if self.config['ema_opt']: self.ema.swap_parameters() self.para_swapped = True def train(self): if self.para_swapped: self.ema.swap_parameters() self.para_swapped = False def update(self, batch_meta, batch_data): self.network.train() labels = batch_data[batch_meta['label']] # print('data size:',batch_data[batch_meta['token_id']].size()) if batch_meta['pairwise']: labels = labels.contiguous().view(-1, batch_meta['pairwise_size'])[:, 0] if self.config['cuda']: y = Variable(labels.cuda(async=True), requires_grad=False) else: y = Variable(labels, requires_grad=False) task_id = batch_meta['task_id'] task_type = batch_meta['task_type'] inputs = batch_data[:batch_meta['input_len']] if len(inputs) == 3: inputs.append(None) inputs.append(None) inputs.append(task_id) # pdb.set_trace() logits = self.mnetwork(*inputs) if batch_meta['pairwise']: logits = logits.view(-1, batch_meta['pairwise_size']) # pdb.set_trace() if task_type > 0: if self.config['answer_relu']: logits = F.relu(logits) loss = F.mse_loss(logits.squeeze(1), y) else: loss = F.cross_entropy(logits, y) if self.config['mediqa_pairloss'] is not None and batch_meta[ 'dataset_name'] in mediqa_name_list: # print(logits) # print(batch_data[batch_meta['rank_label']].size()) # input('ha') logits = logits.squeeze().view(-1, 2) # print(batch_data[batch_meta['rank_label']]) rank_y = batch_data[batch_meta['rank_label']].view(-1, 2) # print(rank_y) if self.config['mediqa_pairloss'] == 'hinge': # print(logits) first_logit, second_logit = logits.split(1, dim=1) # print(first_logit,second_logit) # pdb.set_trace() rank_y = (2 * rank_y - 1).to(torch.float32) rank_y = rank_y[:, 0] pairwise_loss = F.margin_ranking_loss( first_logit.squeeze(1), second_logit.squeeze(1), rank_y, margin=self.config['hinge_lambda']) else: # pdb.set_trace() pairwise_loss = F.cross_entropy(logits, rank_y[:, 1]) # print('pairwise_loss:',pairwise_loss,'mse loss:',loss) loss += pairwise_loss self.train_loss.update(loss.item(), logits.size(0)) self.optimizer.zero_grad() loss.backward() if self.config['global_grad_clipping'] > 0: torch.nn.utils.clip_grad_norm_(self.network.parameters(), self.config['global_grad_clipping']) self.optimizer.step() self.updates += 1 self.update_ema() def predict(self, batch_meta, batch_data): self.network.eval() task_id = batch_meta['task_id'] task_type = batch_meta['task_type'] inputs = batch_data[:batch_meta['input_len']] if len(inputs) == 3: inputs.append(None) inputs.append(None) inputs.append(task_id) score = self.mnetwork(*inputs) gold_label = batch_meta['label'] if batch_meta['pairwise']: score = score.contiguous().view(-1, batch_meta['pairwise_size']) if task_type < 1: score = F.softmax(score, dim=1) score = score.data.cpu() score = score.numpy() predict = np.zeros(score.shape, dtype=int) if task_type < 1: positive = np.argmax(score, axis=1) for idx, pos in enumerate(positive): predict[idx, pos] = 1 predict = predict.reshape(-1).tolist() score = score.reshape(-1).tolist() return score, predict, batch_meta['true_label'] else: if task_type < 1: score = F.softmax(score, dim=1) # pdb.set_trace() score = score.data.cpu() score = score.numpy() if task_type < 1: predict = np.argmax(score, axis=1).tolist() else: predict = np.greater( score, 2.0 + self.config['mediqa_score_offset']).astype(int) gold_label = np.greater( batch_meta['label'], 2.00001 + self.config['mediqa_score_offset']).astype(int) predict = predict.reshape(-1).tolist() gold_label = gold_label.reshape(-1).tolist() # print('predict:',predict,score) score = score.reshape(-1).tolist() return score, predict, gold_label def save(self, filename): network_state = dict([(k, v.cpu()) for k, v in self.network.state_dict().items()]) ema_state = dict([ (k, v.cpu()) for k, v in self.ema.model.state_dict().items() ]) if self.ema is not None else dict() params = { 'state': network_state, 'optimizer': self.optimizer.state_dict(), 'ema': ema_state, 'config': self.config, } torch.save(params, filename) logger.info('model saved to {}'.format(filename)) def cuda(self): self.network.cuda() if self.config['ema_opt']: self.ema.cuda()
[n for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay_rate': 0.01 }, { 'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'names': [n for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay_rate': 0.0 }] args.steps_per_epoch = sum(num_batchs_per_task) args.total_steps = args.steps_per_epoch * args.epoch_num optimizer = BertAdam(params=optimizer_grouped_parameters, lr=args.learning_rate, warmup=args.warmup, t_total=args.total_steps, max_grad_norm=args.clip_grad, schedule=args.schedule) logging.info('Loading graph and entity linking...') graph = pickle.load(open('graph/graph.pkl', 'rb')) entity_linking = pickle.load(open('graph/entity_linking.pkl', 'rb')) if args.do_train_and_eval: # Train and evaluate best_acc = 0 for epoch in range(args.epoch_num): ## Train model.train() t = trange(args.steps_per_epoch, desc='Epoch {} -Train'.format(epoch))
def main(): parser = argparse.ArgumentParser(fromfile_prefix_chars="@") parser.add_argument("--pregenerated_data", type=Path, required=True, help="The input train corpus.") parser.add_argument("--epochs", type=int, required=True) parser.add_argument("--bert_model", type=str, required=True) parser.add_argument("--bert_config_file", type=str, default="bert_config.json") parser.add_argument("--vocab_file", type=str, default="senti_vocab.txt") parser.add_argument('--output_dir', type=Path, required=True) parser.add_argument("--model_name", type=str, default="senti_base_model") parser.add_argument( "--reduce_memory", action="store_true", help= "Store training data as on-disc memmaps to massively reduce memory usage" ) parser.add_argument("--world_size", type=int, default=4) parser.add_argument("--start_rank", type=int, default=0) parser.add_argument("--server", type=str, default="tcp://127.0.0.1:1234") parser.add_argument("--load_model", action="store_true") parser.add_argument("--load_model_name", type=str, default="large_model") parser.add_argument("--save_step", type=int, default=100000) parser.add_argument("--train_batch_size", default=4, type=int, help="Total batch size for training.") parser.add_argument("--learning_rate", default=1e-4, type=float, help="The initial learning rate for Adam.") parser.add_argument( "--warmup_proportion", default=0.1, type=float, help= "Proportion of training to perform linear learning rate warmup for. " "E.g., 0.1 = 10%% of training.") parser.add_argument("--no_cuda", action='store_true', help="Whether not to use CUDA when available") parser.add_argument( "--do_lower_case", action='store_true', help= "Whether to lower case the input text. True for uncased models, False for cased models." ) parser.add_argument("--local_rank", type=int, default=-1, help="local_rank for distributed training on gpus") parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") parser.add_argument( '--gradient_accumulation_steps', type=int, default=1, help= "Number of updates steps to accumualte before performing a backward/update pass." ) parser.add_argument( '--fp16', action='store_true', help="Whether to use 16-bit float precision instead of 32-bit") parser.add_argument( '--loss_scale', type=float, default=0, help= "Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n" "0 (default value): dynamic loss scaling.\n" "Positive power of 2: static loss scaling value.\n") args = parser.parse_args() assert args.pregenerated_data.is_dir(), \ "--pregenerated_data should point to the folder of files made by pregenerate_training_data.py!" print("local_rank : ", args.local_rank) samples_per_epoch = [] for i in range(args.epochs): epoch_file = args.pregenerated_data / f"epoch_{i}.json" metrics_file = args.pregenerated_data / f"epoch_{i}_metrics.json" if epoch_file.is_file() and metrics_file.is_file(): metrics = json.loads(metrics_file.read_text()) samples_per_epoch.append(metrics['num_training_examples']) else: if i == 0: exit("No training data was found!") print( f"Warning! There are fewer epochs of pregenerated data ({i}) than training epochs ({args.epochs})." ) print( "This script will loop over the available data, but training diversity may be negatively impacted." ) num_data_epochs = i break else: num_data_epochs = args.epochs if args.local_rank == -1 or args.no_cuda: device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") n_gpu = torch.cuda.device_count() else: torch.cuda.set_device(args.local_rank) device = torch.device("cuda", args.local_rank) n_gpu = 1 # Initializes the distributed backend which will take care of sychronizing nodes/GPUs torch.distributed.init_process_group(backend='nccl', init_method=args.server, rank=args.local_rank + args.start_rank, world_size=args.world_size) logger.info( "device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}". format(device, n_gpu, bool(args.local_rank != -1), args.fp16)) if args.gradient_accumulation_steps < 1: raise ValueError( "Invalid gradient_accumulation_steps parameter: {}, should be >= 1" .format(args.gradient_accumulation_steps)) args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if n_gpu > 0: torch.cuda.manual_seed_all(args.seed) if args.output_dir.is_dir() and list(args.output_dir.iterdir()): logger.warning( f"Output directory ({args.output_dir}) already exists and is not empty!" ) args.output_dir.mkdir(parents=True, exist_ok=True) tokenizer = Tokenizer( os.path.join(args.bert_model, "senti_vocab.txt"), os.path.join(args.bert_model, "RoBERTa_Sentiment_kor")) total_train_examples = 0 for i in range(args.epochs): # The modulo takes into account the fact that we may loop over limited epochs of data total_train_examples += samples_per_epoch[i % len(samples_per_epoch)] num_train_optimization_steps = math.ceil(total_train_examples / args.train_batch_size / args.gradient_accumulation_steps) if args.local_rank != -1: num_train_optimization_steps = math.ceil( num_train_optimization_steps / torch.distributed.get_world_size()) # Prepare model config = BertConfig.from_json_file( os.path.join(args.bert_model, args.bert_config_file)) logger.info('{}'.format(config)) ############################################### # Load Model if args.load_model: load_model_name = os.path.join(args.output_dir, args.load_model_name) model = BertForPreTraining.from_pretrained( args.bert_model, state_dict=torch.load(load_model_name)["state_dict"]) else: model = BertForPreTraining(config) ############################################### if args.fp16: model.half() model.to(device) if args.local_rank != -1: try: from apex.parallel import DistributedDataParallel as DDP model = DDP(model) except ImportError: from torch.nn.parallel import DistributedDataParallel as DDP model = DDP(model, device_ids=[args.local_rank], output_device=args.local_rank) elif n_gpu > 1: model = torch.nn.DataParallel(model) param_optimizer = list(model.named_parameters()) no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [{ 'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01 }, { 'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0 }] if args.fp16: try: from apex.optimizers import FP16_Optimizer from apex.optimizers import FusedAdam except ImportError: raise ImportError( "Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training." ) optimizer = FusedAdam(optimizer_grouped_parameters, lr=args.learning_rate, bias_correction=False, max_grad_norm=1.0) if args.loss_scale == 0: optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True) else: optimizer = FP16_Optimizer(optimizer, static_loss_scale=args.loss_scale) warmup_linear = WarmupLinearSchedule( warmup=args.warmup_proportion, t_total=num_train_optimization_steps) else: optimizer = BertAdam(optimizer_grouped_parameters, lr=args.learning_rate, warmup=args.warmup_proportion, t_total=num_train_optimization_steps) epoch0 = 0 global_step = 0 if args.load_model: ############################################### # Load Model logger.info(f"***** Load Model {args.load_model_name} *****") loaded_states = torch.load(os.path.join(args.output_dir, args.load_model_name), map_location=device) optimizer.load_state_dict(loaded_states["optimizer"]) regex = re.compile(r'\d+epoch') epoch0 = int( regex.findall(args.load_model_name)[-1].replace('epoch', '')) logger.info('extract {} -> epoch0 : {}'.format(args.load_model_name, epoch0)) ############################################### logger.info("***** Running training *****") logger.info(f" Num examples = {total_train_examples}") logger.info(" Batch size = %d", args.train_batch_size) logger.info(" Num steps = %d", num_train_optimization_steps) model.train() # model.eval() for epoch in range(epoch0, args.epochs): epoch_dataset = PregeneratedDataset( epoch=epoch, training_path=args.pregenerated_data, tokenizer=tokenizer, num_data_epochs=num_data_epochs, reduce_memory=args.reduce_memory) if args.local_rank == -1: train_sampler = RandomSampler(epoch_dataset) else: train_sampler = DistributedSampler(epoch_dataset) train_dataloader = DataLoader(epoch_dataset, sampler=train_sampler, batch_size=args.train_batch_size) tr_loss = 0 nb_tr_examples, nb_tr_steps = 0, 0 with tqdm(total=len(train_dataloader), desc='training..') as pbar: for step, batch in enumerate(train_dataloader): batch = tuple(t.to(device) for t in batch) input_ids, input_mask, lm_label_ids = batch loss = model(input_ids, input_mask, lm_label_ids) if n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu. if args.gradient_accumulation_steps > 1: loss = loss / args.gradient_accumulation_steps if args.fp16: optimizer.backward(loss) else: loss.backward() tr_loss += loss.item() nb_tr_examples += input_ids.size(0) nb_tr_steps += 1 pbar.update(1) mean_loss = tr_loss * args.gradient_accumulation_steps / nb_tr_steps if (step + 1) % 50 == 0: pbar.set_description( "Epoch = {}, global_step = {}, loss = {:.5f}".format( epoch, global_step + 1, mean_loss)) logger.info( "Epoch = {}, global_step = {}, loss = {:.5f}".format( epoch, global_step + 1, mean_loss)) if (step + 1) % args.gradient_accumulation_steps == 0: if args.fp16: # modify learning rate with special warm up BERT uses # if args.fp16 is False, BertAdam is used that handles this automatically lr_this_step = args.learning_rate * warmup_linear.get_lr( global_step, args.warmup_proportion) for param_group in optimizer.param_groups: param_group['lr'] = lr_this_step optimizer.step() optimizer.zero_grad() global_step += 1 if (step + 1) % args.save_step == 0: if args.local_rank == -1 or args.local_rank == 0: logger.info( "** ** * Saving {} - step model ** ** * ".format( global_step)) output_model_file = os.path.join( args.output_dir, args.model_name + "_{}step".format(global_step)) model_to_save = model.module if hasattr( model, 'module') else model state = { "state_dict": model_to_save.state_dict(), "optimizer": optimizer.state_dict() } torch.save(state, output_model_file) if args.local_rank == -1 or args.local_rank == 0: logger.info( "** ** * Saving {} - epoch model ** ** * ".format(epoch)) output_model_file = os.path.join( args.output_dir, args.model_name + "_{}epoch".format(epoch + 1)) model_to_save = model.module if hasattr(model, 'module') else model state = { "state_dict": model_to_save.state_dict(), "optimizer": optimizer.state_dict() } torch.save(state, output_model_file)
def main(*_, **kwargs): use_cuda = torch.cuda.is_available() and kwargs["device"] >= 0 device = torch.device("cuda:" + str(kwargs["device"]) if use_cuda else "cpu") if use_cuda: torch.cuda.set_device(device) kwargs["use_cuda"] = use_cuda neptune.create_experiment( name="bert-span-parser", upload_source_files=[], params={ k: str(v) if isinstance(v, bool) else v for k, v in kwargs.items() }, ) logger.info("Settings: {}", json.dumps(kwargs, indent=2, ensure_ascii=False)) # For reproducibility os.environ["PYTHONHASHSEED"] = str(kwargs["seed"]) random.seed(kwargs["seed"]) np.random.seed(kwargs["seed"]) torch.manual_seed(kwargs["seed"]) torch.cuda.manual_seed_all(kwargs["seed"]) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False # Prepare and load data tokenizer = BertTokenizer.from_pretrained(kwargs["bert_model"], do_lower_case=False) logger.info("Loading data...") train_treebank = load_trees(kwargs["train_file"]) dev_treebank = load_trees(kwargs["dev_file"]) test_treebank = load_trees(kwargs["test_file"]) logger.info( "Loaded {:,} train, {:,} dev, and {:,} test examples!", len(train_treebank), len(dev_treebank), len(test_treebank), ) logger.info("Preprocessing data...") train_parse = [tree.convert() for tree in train_treebank] train_sentences = [[(leaf.tag, leaf.word) for leaf in tree.leaves()] for tree in train_parse] dev_sentences = [[(leaf.tag, leaf.word) for leaf in tree.leaves()] for tree in dev_treebank] test_sentences = [[(leaf.tag, leaf.word) for leaf in tree.leaves()] for tree in test_treebank] logger.info("Data preprocessed!") logger.info("Preparing data for training...") tags = [] labels = [] for tree in train_parse: nodes = [tree] while nodes: node = nodes.pop() if isinstance(node, InternalParseNode): labels.append(node.label) nodes.extend(reversed(node.children)) else: tags.append(node.tag) tag_encoder = LabelEncoder() tag_encoder.fit(tags, reserved_labels=["[PAD]", "[UNK]"]) label_encoder = LabelEncoder() label_encoder.fit(labels, reserved_labels=[()]) logger.info("Data prepared!") # Settings num_train_optimization_steps = kwargs["num_epochs"] * ( (len(train_parse) - 1) // kwargs["batch_size"] + 1) kwargs["batch_size"] //= kwargs["gradient_accumulation_steps"] logger.info("Creating dataloaders for training...") train_dataloader, train_features = create_dataloader( sentences=train_sentences, batch_size=kwargs["batch_size"], tag_encoder=tag_encoder, tokenizer=tokenizer, is_eval=False, ) dev_dataloader, dev_features = create_dataloader( sentences=dev_sentences, batch_size=kwargs["batch_size"], tag_encoder=tag_encoder, tokenizer=tokenizer, is_eval=True, ) test_dataloader, test_features = create_dataloader( sentences=test_sentences, batch_size=kwargs["batch_size"], tag_encoder=tag_encoder, tokenizer=tokenizer, is_eval=True, ) logger.info("Dataloaders created!") # Initialize model model = ChartParser.from_pretrained( kwargs["bert_model"], tag_encoder=tag_encoder, label_encoder=label_encoder, lstm_layers=kwargs["lstm_layers"], lstm_dim=kwargs["lstm_dim"], tag_embedding_dim=kwargs["tag_embedding_dim"], label_hidden_dim=kwargs["label_hidden_dim"], dropout_prob=kwargs["dropout_prob"], ) model.to(device) # Prepare optimizer param_optimizers = list(model.named_parameters()) if kwargs["freeze_bert"]: for p in model.bert.parameters(): p.requires_grad = False param_optimizers = [(n, p) for n, p in param_optimizers if p.requires_grad] # Hack to remove pooler, which is not used thus it produce None grad that break apex param_optimizers = [n for n in param_optimizers if "pooler" not in n[0]] no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"] optimizer_grouped_parameters = [ { "params": [ p for n, p in param_optimizers if not any(nd in n for nd in no_decay) ], "weight_decay": 0.01, }, { "params": [ p for n, p in param_optimizers if any(nd in n for nd in no_decay) ], "weight_decay": 0.0, }, ] optimizer = BertAdam( optimizer_grouped_parameters, lr=kwargs["learning_rate"], warmup=kwargs["warmup_proportion"], t_total=num_train_optimization_steps, ) if kwargs["fp16"]: model, optimizer = amp.initialize(model, optimizer, opt_level="O1") pretrained_model_file = os.path.join(kwargs["output_dir"], MODEL_FILENAME) if kwargs["do_eval"]: assert os.path.isfile( pretrained_model_file), "Pretrained model file does not exist!" logger.info("Loading pretrained model from {}", pretrained_model_file) # Load model from file params = torch.load(pretrained_model_file, map_location=device) model.load_state_dict(params["model"]) logger.info( "Loaded pretrained model (Epoch: {:,}, Fscore: {:.2f})", params["epoch"], params["fscore"], ) eval_score = eval( model=model, eval_dataloader=test_dataloader, eval_features=test_features, eval_trees=test_treebank, eval_sentences=test_sentences, tag_encoder=tag_encoder, device=device, ) neptune.send_metric("test_eval_precision", eval_score.precision()) neptune.send_metric("test_eval_recall", eval_score.recall()) neptune.send_metric("test_eval_fscore", eval_score.fscore()) tqdm.write("Evaluation score: {}".format(str(eval_score))) else: # Training phase global_steps = 0 start_epoch = 0 best_dev_fscore = 0 if kwargs["preload"] or kwargs["resume"]: assert os.path.isfile( pretrained_model_file), "Pretrained model file does not exist!" logger.info("Resuming model from {}", pretrained_model_file) # Load model from file params = torch.load(pretrained_model_file, map_location=device) model.load_state_dict(params["model"]) if kwargs["resume"]: optimizer.load_state_dict(params["optimizer"]) torch.cuda.set_rng_state_all([ state.cpu() for state in params["torch_cuda_random_state_all"] ]) torch.set_rng_state(params["torch_random_state"].cpu()) np.random.set_state(params["np_random_state"]) random.setstate(params["random_state"]) global_steps = params["global_steps"] start_epoch = params["epoch"] + 1 best_dev_fscore = params["fscore"] else: assert not os.path.isfile( pretrained_model_file ), "Please remove or move the pretrained model file to another place!" for epoch in trange(start_epoch, kwargs["num_epochs"], desc="Epoch"): model.train() train_loss = 0 num_train_steps = 0 for step, (indices, *_) in enumerate( tqdm(train_dataloader, desc="Iteration")): ids, attention_masks, tags, sections, trees, sentences = prepare_batch_input( indices=indices, features=train_features, trees=train_parse, sentences=train_sentences, tag_encoder=tag_encoder, device=device, ) loss = model( ids=ids, attention_masks=attention_masks, tags=tags, sections=sections, sentences=sentences, gold_trees=trees, ) if kwargs["gradient_accumulation_steps"] > 1: loss /= kwargs["gradient_accumulation_steps"] if kwargs["fp16"]: with amp.scale_loss(loss, optimizer) as scaled_loss: scaled_loss.backward() else: loss.backward() train_loss += loss.item() num_train_steps += 1 if (step + 1) % kwargs["gradient_accumulation_steps"] == 0: optimizer.step() optimizer.zero_grad() global_steps += 1 # Write logs neptune.send_metric("train_loss", epoch, train_loss / num_train_steps) neptune.send_metric("global_steps", epoch, global_steps) tqdm.write( "Epoch: {:,} - Train loss: {:.4f} - Global steps: {:,}".format( epoch, train_loss / num_train_steps, global_steps)) # Evaluate eval_score = eval( model=model, eval_dataloader=dev_dataloader, eval_features=dev_features, eval_trees=dev_treebank, eval_sentences=dev_sentences, tag_encoder=tag_encoder, device=device, ) neptune.send_metric("eval_precision", epoch, eval_score.precision()) neptune.send_metric("eval_recall", epoch, eval_score.recall()) neptune.send_metric("eval_fscore", epoch, eval_score.fscore()) tqdm.write("Epoch: {:,} - Evaluation score: {}".format( epoch, str(eval_score))) # Save best model if eval_score.fscore() > best_dev_fscore: best_dev_fscore = eval_score.fscore() tqdm.write("** Saving model...") os.makedirs(kwargs["output_dir"], exist_ok=True) torch.save( { "epoch": epoch, "global_steps": global_steps, "fscore": best_dev_fscore, "random_state": random.getstate(), "np_random_state": np.random.get_state(), "torch_random_state": torch.get_rng_state(), "torch_cuda_random_state_all": torch.cuda.get_rng_state_all(), "optimizer": optimizer.state_dict(), "model": (model.module if hasattr(model, "module") else model).state_dict(), }, pretrained_model_file, ) tqdm.write( "** Best evaluation fscore: {:.2f}".format(best_dev_fscore))
param_optimizer = list(model.named_parameters()) no_decay = ['bias', 'gamma', 'beta'] optimizer_grouped_parameters = [ {'params': [p for n, p in param_optimizer if n not in no_decay], 'weight_decay_rate': 0.01}, {'params': [p for n, p in param_optimizer if n in no_decay], 'weight_decay_rate': 0.0} ] num_train_steps = None if args.do_train: num_train_steps = int(len(data.train_data) / args.batch_size / args.gradient_accumulation_steps * args.num_train_epochs) args.batch_size = int(args.batch_size / args.gradient_accumulation_steps) * n_gpu optimizer = BertAdam(optimizer_grouped_parameters, lr=args.learning_rate, warmup=args.warmup_proportion, t_total=num_train_steps) ## Using half precision for faster training if args.fp16: try: from apex import amp except ImportError: raise ImportError("Haven't install apex!!!") model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_level) # For distributed training if args.local_rank != -1: model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], output_device=args.local_rank) if n_gpu > 1: model = torch.nn.DataParallel(model)
def main(): parser = argparse.ArgumentParser() parser.add_argument( "--bert_model", default=None, type=str, required=True, help="Bert pre-trained model selected in the list: bert-base-uncased, " "bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, " "bert-base-multilingual-cased, bert-base-chinese.") parser.add_argument( "--output_dir", default=None, type=str, help="The output directory where the model checkpoints will be written." ) parser.add_argument("--train_file", default=None, type=str) parser.add_argument("--val_file", default=None, type=str) parser.add_argument("--test_file", default=None, type=str) parser.add_argument("--test_output", default=None, type=str) parser.add_argument("--label_vocab", default=None, type=str, required=True) parser.add_argument("--punc_set", default='PU', type=str) parser.add_argument("--has_confidence", action='store_true') parser.add_argument("--only_save_bert", action='store_true') parser.add_argument("--arc_space", default=512, type=int) parser.add_argument("--type_space", default=128, type=int) parser.add_argument("--log_file", default=None, type=str) ## Other parameters parser.add_argument( "--max_seq_length", default=128, type=int, help= "The maximum total input sequence length after WordPiece tokenization. \n" "Sequences longer than this will be truncated, and sequences shorter \n" "than this will be padded.") parser.add_argument("--do_train", action='store_true', help="Whether to run training.") parser.add_argument("--do_eval", action='store_true', help="Whether to run eval on the dev set.") parser.add_argument("--do_predict", action='store_true', help="Whether to run predict on the test set.") parser.add_argument("--do_greedy_predict", action='store_true', help="Whether to run predict on the test set.") parser.add_argument("--do_ensemble_predict", action='store_true', help="Whether to run predict on the test set.") parser.add_argument( "--do_lower_case", action='store_true', help="Set this flag if you are using an uncased model.") parser.add_argument("--train_batch_size", default=32, type=int, help="Total batch size for training.") parser.add_argument("--eval_batch_size", default=8, type=int, help="Total batch size for eval.") parser.add_argument("--test_batch_size", default=8, type=int, help="Total batch size for test.") parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument("--num_train_epochs", default=3.0, type=float, help="Total number of training epochs to perform.") parser.add_argument( "--warmup_proportion", default=0.1, type=float, help= "Proportion of training to perform linear learning rate warmup for. " "E.g., 0.1 = 10%% of training.") parser.add_argument("--no_cuda", action='store_true', help="Whether not to use CUDA when available") parser.add_argument("--local_rank", type=int, default=-1, help="local_rank for distributed training on gpus") parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") parser.add_argument( '--gradient_accumulation_steps', type=int, default=1, help= "Number of updates steps to accumulate before performing a backward/update pass." ) parser.add_argument( '--fp16', action='store_true', help="Whether to use 16-bit float precision instead of 32-bit") parser.add_argument( '--loss_scale', type=float, default=0, help= "Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n" "0 (default value): dynamic loss scaling.\n" "Positive power of 2: static loss scaling value.\n") args = parser.parse_args() if args.log_file is None: logging.basicConfig( format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO) else: logging.basicConfig( format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', filename=args.log_file, filemode='w', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO) if args.local_rank == -1 or args.no_cuda: device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") n_gpu = torch.cuda.device_count() else: torch.cuda.set_device(args.local_rank) device = torch.device("cuda", args.local_rank) n_gpu = 1 # Initializes the distributed backend which will take care of sychronizing nodes/GPUs torch.distributed.init_process_group(backend='nccl') logger.info( "device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}". format(device, n_gpu, bool(args.local_rank != -1), args.fp16)) if args.gradient_accumulation_steps < 1: raise ValueError( "Invalid gradient_accumulation_steps parameter: {}, should be >= 1" .format(args.gradient_accumulation_steps)) args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if n_gpu > 0: torch.cuda.manual_seed_all(args.seed) if not args.do_train and not args.do_predict and not args.do_greedy_predict and not args.do_ensemble_predict: raise ValueError( "At least one of `do_train` or `do_predict` must be True.") if args.do_train: assert args.output_dir is not None if args.do_train and os.path.exists(args.output_dir) and os.listdir( args.output_dir): raise ValueError( "Output directory ({}) already exists and is not empty.".format( args.output_dir)) if args.do_train and not os.path.exists(args.output_dir): os.makedirs(args.output_dir) label_vocab, label_vocab2idx = load_label_vocab(args.label_vocab) punc_set = set( args.punc_set.split(',')) if args.punc_set is not None else None train_examples = None num_train_optimization_steps = None if args.do_train: assert args.train_file is not None train_examples = read_conll_examples( args.train_file, is_training=True, has_confidence=args.has_confidence) num_train_optimization_steps = int( len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps) * args.num_train_epochs if args.local_rank != -1: num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size( ) if args.do_train or args.do_predict or args.do_greedy_predict: # load the pretrained model tokenizer = BertTokenizer.from_pretrained( args.bert_model, do_lower_case=args.do_lower_case) model = BertForDependencyParsing.from_pretrained( args.bert_model, cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed_{}'.format(args.local_rank)), arc_space=args.arc_space, type_space=args.type_space, num_labels=len(label_vocab)) if args.fp16: model.half() model.to(device) if args.local_rank != -1: try: from apex.parallel import DistributedDataParallel as DDP except ImportError: raise ImportError( "Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training." ) model = DDP(model) elif n_gpu > 1: model = torch.nn.DataParallel(model) # parser = model.module if hasattr(model, 'module') else model elif args.do_ensemble_predict: bert_models = args.bert_model.split(',') assert len(bert_models) > 1 tokenizer = BertTokenizer.from_pretrained( bert_models[0], do_lower_case=args.do_lower_case) models = [] for bm in bert_models: model = BertForDependencyParsing.from_pretrained( bm, cache_dir=os.path.join( str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed_{}'.format(args.local_rank)), arc_space=args.arc_space, type_space=args.type_space, num_labels=len(label_vocab)) model.to(device) model.eval() models.append(model) parser = models[0].module if hasattr(models[0], 'module') else models[0] # Prepare optimizer if args.do_train: param_optimizer = list(model.named_parameters()) # hack to remove pooler, which is not used # thus it produce None grad that break apex # !!! NOTE why? param_optimizer = [n for n in param_optimizer if 'pooler' not in n[0]] no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [{ 'params': [ p for n, p in param_optimizer if not any(nd in n for nd in no_decay) ], 'weight_decay': 0.01 }, { 'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0 }] if args.fp16: try: from apex.optimizers import FP16_Optimizer from apex.optimizers import FusedAdam except ImportError: raise ImportError( "Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training." ) optimizer = FusedAdam(optimizer_grouped_parameters, lr=args.learning_rate, bias_correction=False, max_grad_norm=1.0) if args.loss_scale == 0: optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True) else: optimizer = FP16_Optimizer(optimizer, static_loss_scale=args.loss_scale) warmup_linear = WarmupLinearSchedule( warmup=args.warmup_proportion, t_total=num_train_optimization_steps) else: optimizer = BertAdam(optimizer_grouped_parameters, lr=args.learning_rate, warmup=args.warmup_proportion, t_total=num_train_optimization_steps) # start training loop if args.do_train: global_step = 0 train_features = convert_examples_to_features( train_examples, tokenizer, args.max_seq_length, label_vocab2idx, True, has_confidence=args.has_confidence) logger.info("***** Running training *****") logger.info(" Num examples = %d", len(train_examples)) logger.info(" Batch size = %d", args.train_batch_size) logger.info(" Num steps = %d", num_train_optimization_steps) all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.float32) all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long) all_lengths = torch.tensor([f.seq_len for f in train_features], dtype=torch.long) all_heads = torch.tensor([f.heads for f in train_features], dtype=torch.long) all_labels = torch.tensor([f.labels for f in train_features], dtype=torch.long) if args.has_confidence: all_confidence = torch.tensor( [f.confidence for f in train_features], dtype=torch.float32) train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_lengths, all_heads, all_labels, all_confidence) else: train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_lengths, all_heads, all_labels) if args.local_rank == -1: train_sampler = RandomSampler(train_data) else: train_sampler = DistributedSampler(train_data) train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size) if args.do_eval: assert args.val_file is not None eval_examples = read_conll_examples(args.val_file, is_training=False, has_confidence=False) eval_features = convert_examples_to_features(eval_examples, tokenizer, args.max_seq_length, label_vocab2idx, False, has_confidence=False) logger.info(" Num examples = %d", len(eval_examples)) logger.info(" Batch size = %d", args.eval_batch_size) all_example_ids = torch.tensor( [f.example_id for f in eval_features], dtype=torch.long) all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long) all_input_mask = torch.tensor( [f.input_mask for f in eval_features], dtype=torch.float32) all_segment_ids = torch.tensor( [f.segment_ids for f in eval_features], dtype=torch.long) all_lengths = torch.tensor([f.seq_len for f in eval_features], dtype=torch.long) eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_lengths, all_example_ids) # Run prediction for full data eval_sampler = SequentialSampler(eval_data) eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size) best_uas = 0 best_las = 0 for epoch in trange(int(args.num_train_epochs), desc="Epoch"): logger.info("Training epoch: {}".format(epoch)) tr_loss = 0 nb_tr_examples, nb_tr_steps = 0, 0 model.train() for step, batch in enumerate( tqdm(train_dataloader, desc="Iteration")): batch = tuple(t.to(device) for t in batch) if args.has_confidence: input_ids, input_mask, segment_ids, lengths, heads, label_ids, confidence = batch else: confidence = None input_ids, input_mask, segment_ids, lengths, heads, label_ids = batch loss = model(input_ids, segment_ids, input_mask, heads, label_ids, confidence) if n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu. if args.fp16 and args.loss_scale != 1.0: # rescale loss for fp16 training # see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html loss = loss * args.loss_scale if args.gradient_accumulation_steps > 1: loss = loss / args.gradient_accumulation_steps tr_loss += loss.item() nb_tr_examples += input_ids.size(0) nb_tr_steps += 1 if args.fp16: optimizer.backward(loss) else: loss.backward() if (step + 1) % args.gradient_accumulation_steps == 0: if args.fp16: # modify learning rate with special warm up BERT uses # if args.fp16 is False, BertAdam is used that handles this automatically lr_this_step = args.learning_rate * warmup_linear.get_lr( global_step, args.warmup_proportion) for param_group in optimizer.param_groups: param_group['lr'] = lr_this_step optimizer.step() optimizer.zero_grad() global_step += 1 if global_step % 100 == 0: logger.info("Training loss: {}, global step: {}".format( tr_loss / nb_tr_steps, global_step)) # we eval every epoch if args.do_eval and (args.local_rank == -1 or torch.distributed.get_rank() == 0): logger.info("***** Running evaluation *****") model.eval() eval_predict_words, eval_predict_postags, eval_predict_heads, eval_predict_labels = [],[],[],[] for input_ids, input_mask, segment_ids, lengths, example_ids in tqdm( eval_dataloader, desc="Evaluating"): example_ids = example_ids.numpy() batch_words = [ eval_features[eid].example.sentence for eid in example_ids ] batch_postags = [ eval_features[eid].example.postags for eid in example_ids ] batch_word_index = [ eval_features[eid].word_index for eid in example_ids ] # token -> word batch_token_starts = [ eval_features[eid].token_starts for eid in example_ids ] # word -> token start batch_heads = [ eval_features[eid].example.heads for eid in example_ids ] input_ids = input_ids.to(device) input_mask = input_mask.to(device) segment_ids = segment_ids.to(device) heads = heads.to(device) label_ids = label_ids.to(device) with torch.no_grad(): # tmp_eval_loss = model(input_ids, segment_ids, input_mask, heads, label_ids) energy = model(input_ids, segment_ids, input_mask) heads_pred, labels_pred = parser.decode_MST( energy.cpu().numpy(), lengths.numpy(), leading_symbolic=0, labeled=True) # we convert the subword dependency parsing to word dependency parsing just the word and token start map pred_heads = [] pred_labels = [] for i in range(len(batch_word_index)): word_index = batch_word_index[i] token_starts = batch_token_starts[i] hpd = [] lpd = [] for j in range(len(token_starts)): if j == 0: #[CLS] continue elif j == len(token_starts) - 1: # [SEP] continue else: hpd.append( word_index[heads_pred[i, token_starts[j]]]) lpd.append( label_vocab[labels_pred[i, token_starts[j]]]) pred_heads.append(hpd) pred_labels.append(lpd) eval_predict_words += batch_words eval_predict_postags += batch_postags eval_predict_heads += pred_heads eval_predict_labels += pred_labels eval_output_file = os.path.join(args.output_dir, 'eval.pred') write_conll_examples(eval_predict_words, eval_predict_postags, eval_predict_heads, eval_predict_labels, eval_output_file) eval_f = os.popen( "python scripts/eval_nlpcc_dp.py " + args.val_file + " " + eval_output_file, "r") result_text = eval_f.read().strip() logger.info("***** Eval results *****") logger.info(result_text) eval_f.close() eval_res = re.findall( r'UAS = \d+/\d+ = ([\d\.]+), LAS = \d+/\d+ = ([\d\.]+)', result_text) assert len(eval_res) > 0 eval_res = eval_res[0] eval_uas = float(eval_res[0]) eval_las = float(eval_res[1]) # save model if best_las < eval_las or (eval_las == best_las and best_uas < eval_uas): best_uas = eval_uas best_las = eval_las logger.info( "new best uas %.2f%% las %.2f%%, saving models.", best_uas, best_las) # Save a trained model, configuration and tokenizer model_to_save = model.module if hasattr( model, 'module') else model # Only save the model it-self # If we save using the predefined names, we can load using `from_pretrained` output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME) output_config_file = os.path.join(args.output_dir, CONFIG_NAME) model_dict = model_to_save.state_dict() if args.only_save_bert: model_dict = { k: v for k, v in model_dict.items() if 'bert.' in k } torch.save(model_dict, output_model_file) model_to_save.config.to_json_file(output_config_file) tokenizer.save_vocabulary(args.output_dir) # start predict if args.do_predict: model.eval() assert args.test_file is not None test_examples = read_conll_examples(args.test_file, is_training=False, has_confidence=False) test_features = convert_examples_to_features(test_examples, tokenizer, args.max_seq_length, label_vocab2idx, False, has_confidence=False) logger.info("***** Running prediction *****") logger.info(" Num examples = %d", len(test_examples)) logger.info(" Batch size = %d", args.test_batch_size) all_example_ids = torch.tensor([f.example_id for f in test_features], dtype=torch.long) all_input_ids = torch.tensor([f.input_ids for f in test_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in test_features], dtype=torch.float32) all_segment_ids = torch.tensor([f.segment_ids for f in test_features], dtype=torch.long) all_lengths = torch.tensor([f.seq_len for f in test_features], dtype=torch.long) test_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_lengths, all_example_ids) # Run prediction for full data test_sampler = SequentialSampler(test_data) test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=args.test_batch_size) test_predict_words, test_predict_postags, test_predict_heads, test_predict_labels = [],[],[],[] for batch_id, batch in enumerate( tqdm(test_dataloader, desc="Predicting")): input_ids, input_mask, segment_ids, lengths, example_ids = batch example_ids = example_ids.numpy() batch_words = [ test_features[eid].example.sentence for eid in example_ids ] batch_postags = [ test_features[eid].example.postags for eid in example_ids ] batch_word_index = [ test_features[eid].word_index for eid in example_ids ] # token -> word batch_token_starts = [ test_features[eid].token_starts for eid in example_ids ] # word -> token start input_ids = input_ids.to(device) input_mask = input_mask.to(device) segment_ids = segment_ids.to(device) lengths = lengths.numpy() with torch.no_grad(): energy = model(input_ids, segment_ids, input_mask) heads_pred, labels_pred = parser.decode_MST(energy.cpu().numpy(), lengths, leading_symbolic=0, labeled=True) pred_heads = [] pred_labels = [] for i in range(len(batch_word_index)): word_index = batch_word_index[i] token_starts = batch_token_starts[i] hpd = [] lpd = [] for j in range(len(token_starts)): if j == 0: #[CLS] continue elif j == len(token_starts) - 1: # [SEP] continue else: hpd.append(word_index[heads_pred[i, token_starts[j]]]) lpd.append(label_vocab[labels_pred[i, token_starts[j]]]) pred_heads.append(hpd) pred_labels.append(lpd) test_predict_words += batch_words test_predict_postags += batch_postags test_predict_heads += pred_heads test_predict_labels += pred_labels assert args.test_output is not None write_conll_examples(test_predict_words, test_predict_postags, test_predict_heads, test_predict_labels, args.test_output) if args.do_greedy_predict: model.eval() assert args.test_file is not None test_examples = read_conll_examples(args.test_file, is_training=False, has_confidence=False) test_features = convert_examples_to_features(test_examples, tokenizer, args.max_seq_length, label_vocab2idx, False, has_confidence=False) logger.info("***** Running prediction *****") logger.info(" Num examples = %d", len(test_examples)) logger.info(" Batch size = %d", args.test_batch_size) all_example_ids = torch.tensor([f.example_id for f in test_features], dtype=torch.long) all_input_ids = torch.tensor([f.input_ids for f in test_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in test_features], dtype=torch.float32) all_segment_ids = torch.tensor([f.segment_ids for f in test_features], dtype=torch.long) all_lengths = torch.tensor([f.seq_len for f in test_features], dtype=torch.long) test_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_lengths, all_example_ids) # Run prediction for full data test_sampler = SequentialSampler(test_data) test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=args.test_batch_size) test_predict_words, test_predict_postags, test_predict_heads, test_predict_labels = [],[],[],[] for batch_id, batch in enumerate( tqdm(test_dataloader, desc="Predicting")): input_ids, input_mask, segment_ids, lengths, example_ids = batch example_ids = example_ids.numpy() batch_words = [ test_features[eid].example.sentence for eid in example_ids ] batch_postags = [ test_features[eid].example.postags for eid in example_ids ] batch_word_index = [ test_features[eid].word_index for eid in example_ids ] # token -> word batch_token_starts = [ test_features[eid].token_starts for eid in example_ids ] # word -> token start input_ids = input_ids.to(device) input_mask = input_mask.to(device) segment_ids = segment_ids.to(device) lengths = lengths.numpy() with torch.no_grad(): heads_pred, labels_pred = model(input_ids, segment_ids, input_mask, greedy_inference=True) pred_heads = [] pred_labels = [] for i in range(len(batch_word_index)): word_index = batch_word_index[i] token_starts = batch_token_starts[i] hpd = [] lpd = [] for j in range(len(token_starts)): if j == 0: #[CLS] continue elif j == len(token_starts) - 1: # [SEP] continue else: hpd.append(word_index[heads_pred[i, token_starts[j]]]) lpd.append(label_vocab[labels_pred[i, token_starts[j]]]) pred_heads.append(hpd) pred_labels.append(lpd) test_predict_words += batch_words test_predict_postags += batch_postags test_predict_heads += pred_heads test_predict_labels += pred_labels assert args.test_output is not None write_conll_examples(test_predict_words, test_predict_postags, test_predict_heads, test_predict_labels, args.test_output) if args.do_ensemble_predict: assert args.test_file is not None test_examples = read_conll_examples(args.test_file, is_training=False, has_confidence=False) test_features = convert_examples_to_features(test_examples, tokenizer, args.max_seq_length, label_vocab2idx, False, has_confidence=False) logger.info("***** Running evaluation *****") logger.info(" Num examples = %d", len(test_examples)) logger.info(" Batch size = %d", args.test_batch_size) all_example_ids = torch.tensor([f.example_id for f in test_features], dtype=torch.long) all_input_ids = torch.tensor([f.input_ids for f in test_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in test_features], dtype=torch.float32) all_segment_ids = torch.tensor([f.segment_ids for f in test_features], dtype=torch.long) all_lengths = torch.tensor([f.seq_len for f in test_features], dtype=torch.long) test_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_lengths, all_example_ids) # Run prediction for full data test_sampler = SequentialSampler(test_data) test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=args.test_batch_size) test_predict_words, test_predict_postags, test_predict_heads, test_predict_labels = [],[],[],[] for batch_id, batch in enumerate( tqdm(test_dataloader, desc="Predicting")): input_ids, input_mask, segment_ids, lengths, example_ids = batch example_ids = example_ids.numpy() batch_words = [ test_features[eid].example.sentence for eid in example_ids ] batch_postags = [ test_features[eid].example.postags for eid in example_ids ] batch_word_index = [ test_features[eid].word_index for eid in example_ids ] # token -> word batch_token_starts = [ test_features[eid].token_starts for eid in example_ids ] # word -> token start input_ids = input_ids.to(device) input_mask = input_mask.to(device) segment_ids = segment_ids.to(device) lengths = lengths.numpy() with torch.no_grad(): energy_sum = None for model in models: energy = model(input_ids, segment_ids, input_mask) if energy_sum is None: energy_sum = energy else: energy_sum = energy_sum + energy energy_sum = energy_sum / len(models) heads_pred, labels_pred = parser.decode_MST( energy_sum.cpu().numpy(), lengths, leading_symbolic=0, labeled=True) pred_heads = [] pred_labels = [] for i in range(len(batch_word_index)): word_index = batch_word_index[i] token_starts = batch_token_starts[i] hpd = [] lpd = [] for j in range(len(token_starts)): if j == 0: #[CLS] continue elif j == len(token_starts) - 1: # [SEP] continue else: hpd.append(word_index[heads_pred[i, token_starts[j]]]) lpd.append(label_vocab[labels_pred[i, token_starts[j]]]) pred_heads.append(hpd) pred_labels.append(lpd) test_predict_words += batch_words test_predict_postags += batch_postags test_predict_heads += pred_heads test_predict_labels += pred_labels assert args.test_output is not None write_conll_examples(test_predict_words, test_predict_postags, test_predict_heads, test_predict_labels, args.test_output)
def create_optimizer(model, args, num_train_steps=None, init_spec=None, no_decay=['bias', 'LayerNorm.weight']): # Prepare optimizer if args.fp16: dcnt = torch.cuda.device_count() if args.no_even_grad: param_optimizer = [(n, param.detach().clone().type(torch.cuda.FloatTensor).\ requires_grad_()) for i,(n,param) in enumerate(model.named_parameters())] else: total_size = sum(np.prod(p.size()) for p in model.parameters()) quota = {i: 0 for i in range(dcnt)} quota[0] = total_size // (dcnt * 2) param_optimizer = [] for i, (n, param) in enumerate(model.named_parameters()): ps = np.prod(param.size()) index = list(sorted(quota.items(), key=lambda x: x[1]))[0][0] quota[index] += ps cp = param.clone().type(torch.cuda.FloatTensor).detach().to( 'cuda:{}'.format(index)).requires_grad_() param_optimizer += [(n, cp)] elif args.optimize_on_cpu: param_optimizer = [(n, param.clone().detach().to('cpu').requires_grad_()) \ for n, param in model.named_parameters()] else: param_optimizer = [(n, p) for n, p in model.named_parameters()] group0 = dict(params=[], weight_decay_rate=args.weight_decay, names=[]) group1 = dict(params=[], weight_decay_rate=0.00, names=[]) for (n, p) in param_optimizer: if not any(nd in n for nd in no_decay): group0['params'].append(p) group0['names'].append(n) else: group1['params'].append(p) group1['names'].append(n) optimizer_grouped_parameters = [group0, group1] t_total = num_train_steps optimizer = None if t_total: if args.local_rank != -1: t_total = t_total // torch.distributed.get_world_size() optimizer = BertAdam( optimizer_grouped_parameters, lr=args.learning_rate, b1=args.adam_beta1, b2=args.adam_beta2, v1=args.qhadam_v1, v2=args.qhadam_v2, lr_ends=args.lr_schedule_ends, e=args.epsilon, warmup=args.warmup_proportion if args.warmup_proportion < 1 else args.warmup_proportion / t_total, t_total=t_total, schedule=args.lr_schedule, max_grad_norm=args.max_grad_norm, global_grad_norm=args.global_grad_norm, init_spec=init_spec, weight_decay_rate=args.weight_decay) return optimizer, param_optimizer, t_total
def main(): parser = argparse.ArgumentParser() parser.add_argument("--device", default=None, type=str, required=True, help="The GPU device you will run on.") parser.add_argument( "--features_file", default=None, type=str, required=True, help= "The train features file. Should contain the .csv files (after tokenized) for the task." "Format: example_id,input_ids,input_mask,segment_ids,label\n") parser.add_argument( "--teacher_model", default=None, type=str, help= "The teacher model dir. Should contain the config/vocab/checkpoint file." ) parser.add_argument( "--general_student_model", default=None, type=str, required=True, help="The student model (after general distillation) dir. " "Should contain the config/vocab/checkpoint file.") parser.add_argument( "--output_student_dir", default=None, type=str, required=True, help= "The output directory for the task-specific distilled student models.") parser.add_argument("--cache_file_dir", default='./cache', type=str, required=True, help="The directory where cache the features.") parser.add_argument( "--distill_model", default='simplified', type=str, help="The distill model type, choose in 'standard' and 'simplified'.") parser.add_argument( "--max_seq_length", default=256, type=int, help= "The maximum total input sequence length after WordPiece tokenization." ) parser.add_argument( "--do_lower_case", action='store_true', help="Set this flag if you are using an uncased model.") parser.add_argument("--train_batch_size", default=64, type=int, help="Total batch size for training.") parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument('--weight_decay', '--wd', default=1e-2, type=float, metavar='W', help='weight decay') parser.add_argument("--num_train_epochs", default=2, type=float, help="Total number of training epochs to perform.") parser.add_argument( "--alpha", default=0.5, type=float, help="The weight of soft loss in standard kd method." "Only use when '--distill_model' is set as 'standard'.") parser.add_argument( "--warmup_proportion", default=0.1, type=float, help= "Proportion of training to perform linear learning rate warmup for. " "E.g., 0.1 = 10%% of training.") parser.add_argument("--no_cuda", action='store_true', help="Whether not to use CUDA when available") parser.add_argument('--seed', type=int, default=42, help="Random seed for initialization") parser.add_argument( '--gradient_accumulation_steps', type=int, default=1, help= "Number of updates steps to accumulate before performing a backward/update pass." ) parser.add_argument( '--train_loss_step', type=int, default=1000, help="How many train step to record a training loss. ") parser.add_argument('--save_model_step', type=int, default=3000, help="How many train step to save a student model.") parser.add_argument('--temperature', type=float, default=1., help="The temperature in soft loss.") parser.add_argument( '--fp16', action='store_true', help= "Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit." ) parser.add_argument( '--fp16_opt_level', type=str, default='O1', help= "For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']." "See details at https://nvidia.github.io/apex/amp.html") args = parser.parse_args() logger.info('The args: {}'.format(args)) # Prepare device os.environ["CUDA_VISIBLE_DEVICES"] = args.device device = torch.device( "cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") n_gpu = torch.cuda.device_count() logger.info("device: {} n_gpu: {}".format(device, n_gpu)) # Prepare seed random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if n_gpu > 0: torch.cuda.manual_seed_all(args.seed) # Prepare task settings if os.path.exists(args.output_student_dir) and os.listdir( args.output_student_dir): raise ValueError( "Output directory ({}) already exists and is not empty.".format( args.output_student_dir)) if not os.path.exists(args.output_student_dir): os.makedirs(args.output_student_dir) if not os.path.exists(args.cache_file_dir): os.makedirs(args.cache_file_dir) # For save vocab file for all output models. tokenizer = BertTokenizer.from_pretrained(args.general_student_model, do_lower_case=args.do_lower_case) # Model teacher_model = TinyBertForSequenceClassification.from_pretrained( args.teacher_model, num_labels=2) if args.fp16: teacher_model.half() teacher_model.to(device) student_model = TinyBertForSequenceClassification.from_pretrained( args.general_student_model, num_labels=2) student_model.to(device) # Train Config num_examples, train_dataloader = distill_dataloader( args, RandomSampler, batch_size=args.train_batch_size) if args.gradient_accumulation_steps < 1: raise ValueError( "Invalid gradient_accumulation_steps parameter: {}, should be >= 1" .format(args.gradient_accumulation_steps)) num_train_optimization_steps = int( num_examples / args.train_batch_size / args.gradient_accumulation_steps) * args.num_train_epochs logger.info("***** Running Distilling *****") logger.info(" Num examples = %d", num_examples) logger.info(" Batch size = %d", args.train_batch_size) logger.info(" Num steps = %d", num_train_optimization_steps) # Prepare optimizer param_optimizer = list(student_model.named_parameters()) size = 0 for n, p in student_model.named_parameters(): logger.info('n: {}'.format(n)) size += p.nelement() logger.info('Total parameters of student_model: {}'.format(size)) no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [{ 'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay }, { 'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0 }] schedule = 'warmup_linear' optimizer = BertAdam(optimizer_grouped_parameters, schedule=schedule, lr=args.learning_rate, warmup=args.warmup_proportion, t_total=num_train_optimization_steps) if args.fp16: try: from apex import amp except ImportError: raise ImportError( "Please install apex from https://www.github.com/nvidia/apex to use fp16 training." ) student_model, optimizer = amp.initialize( student_model, optimizer, opt_level=args.fp16_opt_level) logger.info('FP16 is activated, use amp') else: logger.info('FP16 is not activated, only use BertAdam') if n_gpu > 1: student_model = torch.nn.DataParallel(student_model) teacher_model = torch.nn.DataParallel(teacher_model) # Prepare loss functions loss_mse = MSELoss() def soft_cross_entropy(predicts, targets): student_likelihood = torch.nn.functional.log_softmax(predicts, dim=-1) targets_prob = torch.nn.functional.softmax(targets, dim=-1) return (-targets_prob * student_likelihood).mean() # Train global_step = 0 output_loss_file = os.path.join(args.output_student_dir, "train_loss.txt") tr_loss = 0. tr_att_loss = 0. tr_rep_loss = 0. tr_cls_loss = 0. for epoch in trange(int(args.num_train_epochs), desc="Epoch"): student_model.train() for step, batch in enumerate( tqdm(train_dataloader, desc="Iteration", ascii=True)): batch = tuple(t.to(device) for t in batch) input_ids, input_mask, segment_ids, label_ids = batch if input_ids.size()[0] != args.train_batch_size: continue student_logits, student_atts, student_reps = student_model( input_ids, segment_ids, input_mask, is_student=True) with torch.no_grad(): teacher_logits, teacher_atts, teacher_reps = teacher_model( input_ids, segment_ids, input_mask) soft_loss = soft_cross_entropy(student_logits / args.temperature, teacher_logits / args.temperature) hard_loss = torch.nn.functional.cross_entropy(student_logits, label_ids, reduction='mean') if args.distill_model == 'standard': cls_loss = args.alpha * soft_loss + (1 - args.alpha) * hard_loss tr_cls_loss += cls_loss.item() loss = cls_loss elif args.distill_model == 'simplified': teacher_layer_num = len(teacher_atts) student_layer_num = len(student_atts) assert teacher_layer_num % student_layer_num == 0 layers_per_block = int(teacher_layer_num / student_layer_num) new_teacher_atts = [ teacher_atts[i * layers_per_block + layers_per_block - 1] for i in range(student_layer_num) ] att_loss = 0. rep_loss = 0. # attention loss for student_att, teacher_att in zip(student_atts, new_teacher_atts): student_att = torch.where( student_att <= -1e2, torch.zeros_like(student_att).to(device), student_att) teacher_att = torch.where( teacher_att <= -1e2, torch.zeros_like(teacher_att).to(device), teacher_att) tmp_loss = loss_mse(student_att, teacher_att) att_loss += tmp_loss # hidden states loss new_teacher_reps = [ teacher_reps[i * layers_per_block] for i in range(student_layer_num + 1) ] new_student_reps = student_reps for student_rep, teacher_rep in zip(new_student_reps, new_teacher_reps): tmp_loss = loss_mse(student_rep, teacher_rep) rep_loss += tmp_loss tr_att_loss += att_loss.item() tr_rep_loss += rep_loss.item() # classification loss cls_loss = soft_loss + hard_loss tr_cls_loss += cls_loss.item() # total loss loss = rep_loss + att_loss + cls_loss else: raise NotImplementedError if n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu. if args.gradient_accumulation_steps > 1: loss = loss / args.gradient_accumulation_steps if args.fp16: with amp.scale_loss(loss, optimizer) as scaled_loss: scaled_loss.backward() else: loss.backward() tr_loss += loss.item() if (step + 1) % args.gradient_accumulation_steps == 0: optimizer.step() optimizer.zero_grad() global_step += 1 if global_step % args.train_loss_step == 0: loss = tr_loss / args.train_loss_step cls_loss = tr_cls_loss / args.train_loss_step att_loss = tr_att_loss / args.train_loss_step rep_loss = tr_rep_loss / args.train_loss_step loss_dict = {} loss_dict['global_step'] = global_step loss_dict['cls_loss'] = cls_loss loss_dict['att_loss'] = att_loss loss_dict['rep_loss'] = rep_loss loss_dict['loss'] = loss write_loss_to_file(loss_dict, output_loss_file) tr_loss = 0. tr_att_loss = 0. tr_rep_loss = 0. tr_cls_loss = 0. if global_step % args.save_model_step == 0: logger.info("***** Save model *****") model_to_save = student_model.module if hasattr( student_model, 'module') else student_model model_name = WEIGHTS_NAME checkpoint_name = 'checkpoint-' + str(global_step) output_model_dir = os.path.join(args.output_dir, checkpoint_name) if not os.path.exists(output_model_dir): os.makedirs(output_model_dir) output_model_file = os.path.join(output_model_dir, model_name) output_config_file = os.path.join(output_model_dir, CONFIG_NAME) torch.save(model_to_save.state_dict(), output_model_file) model_to_save.config.to_json_file(output_config_file) tokenizer.save_vocabulary(output_model_dir) if os.path.exists(args.cache_file_dir): import shutil shutil.rmtree(args.cache_file_dir)