def main(): parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) # Path options. parser.add_argument("--dataset_path", type=str, default="dataset.pt", help="Path of the preprocessed dataset.") parser.add_argument("--vocab_path", default=None, type=str, help="Path of the vocabulary file.") parser.add_argument("--spm_model_path", default=None, type=str, help="Path of the sentence piece model.") parser.add_argument("--tgt_vocab_path", default=None, type=str, help="Path of the target vocabulary file.") parser.add_argument("--tgt_spm_model_path", default=None, type=str, help="Path of the target sentence piece model.") parser.add_argument("--pretrained_model_path", type=str, default=None, help="Path of the pretrained model.") parser.add_argument("--output_model_path", type=str, required=True, help="Path of the output model.") parser.add_argument("--config_path", type=str, default="models/bert/base_config.json", help="Config file of model hyper-parameters.") # Training and saving options. parser.add_argument("--total_steps", type=int, default=100000, help="Total training steps.") parser.add_argument("--save_checkpoint_steps", type=int, default=10000, help="Specific steps to save model checkpoint.") parser.add_argument("--report_steps", type=int, default=100, help="Specific steps to print prompt.") parser.add_argument("--accumulation_steps", type=int, default=1, help="Specific steps to accumulate gradient.") parser.add_argument("--batch_size", type=int, default=32, help="Training batch size. The actual batch_size is [batch_size x world_size x accumulation_steps].") parser.add_argument("--instances_buffer_size", type=int, default=25600, help="The buffer size of instances in memory.") parser.add_argument("--labels_num", type=int, required=False, help="Number of prediction labels.") parser.add_argument("--dropout", type=float, default=0.1, help="Dropout value.") parser.add_argument("--seed", type=int, default=7, help="Random seed.") # Preprocess options. parser.add_argument("--tokenizer", choices=["bert", "char", "space", "xlmroberta"], default="bert", help="Specify the tokenizer." "Original Google BERT uses bert tokenizer." "Char tokenizer segments sentences into characters." "Space tokenizer segments sentences into words according to space." "Original XLM-RoBERTa uses xlmroberta tokenizer." ) parser.add_argument("--tgt_tokenizer", choices=["bert", "char", "space", "xlmroberta"], default="bert", help="Specify the tokenizer for target side.") # Model options. model_opts(parser) parser.add_argument("--tgt_embedding", choices=["word", "word_pos", "word_pos_seg", "word_sinusoidalpos"], default="word_pos_seg", help="Target embedding type.") parser.add_argument("--decoder", choices=["transformer"], default="transformer", help="Decoder type.") parser.add_argument("--pooling", choices=["mean", "max", "first", "last"], default="first", help="Pooling type.") parser.add_argument("--target", choices=["bert", "lm", "mlm", "bilm", "albert", "seq2seq", "t5", "cls", "prefixlm", "gsg", "bart"], default="bert", help="The training target of the pretraining model.") parser.add_argument("--tie_weights", action="store_true", help="Tie the word embedding and softmax weights.") parser.add_argument("--has_lmtarget_bias", action="store_true", help="Add bias on output_layer for lm target.") parser.add_argument("--deep_init", action="store_true", help="initialize bert model similar to gpt2 model." "scales initialization of projection layers by a " "factor of 1/sqrt(2N). Necessary to train bert " "models larger than BERT-Large.") # Masking options. parser.add_argument("--whole_word_masking", action="store_true", help="Whole word masking.") parser.add_argument("--span_masking", action="store_true", help="Span masking.") parser.add_argument("--span_geo_prob", type=float, default=0.2, help="Hyperparameter of geometric distribution for span masking.") parser.add_argument("--span_max_length", type=int, default=10, help="Max length for span masking.") # Optimizer options. optimization_opts(parser) # GPU options. parser.add_argument("--world_size", type=int, default=1, help="Total number of processes (GPUs) for training.") parser.add_argument("--gpu_ranks", default=[], nargs='+', type=int, help="List of ranks of each process." " Each process has a unique integer rank whose value is in the interval [0, world_size), and runs in a single GPU.") parser.add_argument("--master_ip", default="tcp://localhost:12345", type=str, help="IP-Port of master for training.") parser.add_argument("--backend", choices=["nccl", "gloo"], default="nccl", type=str, help="Distributed backend.") args = parser.parse_args() if args.target == "cls": assert args.labels_num is not None, "Cls target needs the denotation of the number of labels." # Load hyper-parameters from config file. if args.config_path: load_hyperparam(args) ranks_num = len(args.gpu_ranks) if args.world_size > 1: # Multiprocessing distributed mode. assert torch.cuda.is_available(), "No available GPUs." assert ranks_num <= args.world_size, "Started processes exceed `world_size` upper limit." assert ranks_num <= torch.cuda.device_count(), "Started processes exceeds the available GPUs." args.dist_train = True args.ranks_num = ranks_num print("Using distributed mode for training.") elif args.world_size == 1 and ranks_num == 1: # Single GPU mode. assert torch.cuda.is_available(), "No available GPUs." args.gpu_id = args.gpu_ranks[0] assert args.gpu_id < torch.cuda.device_count(), "Invalid specified GPU device." args.dist_train = False args.single_gpu = True print("Using GPU %d for training." % args.gpu_id) else: # CPU mode. assert ranks_num == 0, "GPUs are specified, please check the arguments." args.dist_train = False args.single_gpu = False print("Using CPU mode for training.") trainer.train_and_validate(args)
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) # Path options. parser.add_argument("--dataset_path", type=str, default="dataset.pt", help="Path of the preprocessed dataset.") parser.add_argument("--vocab_path", type=str, required=True, help="Path of the vocabulary file.") parser.add_argument("--pretrained_model_path", type=str, default=None, help="Path of the pretrained model.") parser.add_argument("--output_model_path", type=str, required=True, help="Path of the output model.") parser.add_argument("--config_path", type=str, default=None, help="Config file of model hyper-parameters.") # Training and saving options. parser.add_argument("--total_steps", type=int, default=100000, help="Total training steps.") parser.add_argument("--save_checkpoint_steps", type=int, default=10000, help="Specific steps to save model checkpoint.") parser.add_argument("--report_steps", type=int, default=100, help="Specific steps to print prompt.") parser.add_argument("--accumulation_steps", type=int, default=1, help="Specific steps to accumulate gradient.") parser.add_argument( "--batch_size", type=int, default=32, help= "Training batch size. The actual batch_size is [batch_size x world_size x accumulation_steps]." ) parser.add_argument("--instances_buffer_size", type=int, default=25600, help="The buffer size of instances in memory.") # Model options. parser.add_argument("--emb_size", type=int, default=768, help="Embedding dimension.") parser.add_argument("--hidden_size", type=int, default=768, help="Hidden state dimension.") parser.add_argument("--feedforward_size", type=int, default=3072, help="Feed forward layer dimension.") parser.add_argument("--kernel_size", type=int, default=3, help="Kernel size for CNN.") parser.add_argument("--block_size", type=int, default=2, help="Block size for CNN.") parser.add_argument("--heads_num", type=int, default=12, help="The number of heads in multi-head attention.") parser.add_argument("--layers_num", type=int, default=12, help="The number of encoder layers.") parser.add_argument("--dropout", type=float, default=0.1, help="Dropout value.") parser.add_argument("--seed", type=int, default=7, help="Random seed.") parser.add_argument("--encoder", choices=["bert", "lstm", "gru", \ "cnn", "gatedcnn", "attn", \ "rcnn", "crnn", "gpt", "bilstm"], \ default="bert", help="Encoder type.") parser.add_argument("--bidirectional", action="store_true", help="Specific to recurrent model.") parser.add_argument("--target", choices=["bert", "lm", "cls", "mlm", "bilm"], default="bert", help="The training target of the pretraining model.") parser.add_argument("--labels_num", type=int, default=2, help="Specific to classification target.") # Optimizer options. parser.add_argument("--learning_rate", type=float, default=2e-5, help="Initial learning rate.") parser.add_argument("--warmup", type=float, default=0.1, help="Warm up value.") # Subword options. parser.add_argument("--subword_type", choices=["none", "char"], default="none", help="Subword feature type.") parser.add_argument("--sub_vocab_path", type=str, default="models/sub_vocab.txt", help="Path of the subword vocabulary file.") parser.add_argument("--subencoder", choices=["avg", "lstm", "gru", "cnn"], default="avg", help="Subencoder type.") parser.add_argument("--sub_layers_num", type=int, default=2, help="The number of subencoder layers.") # GPU options. parser.add_argument("--world_size", type=int, default=1, help="Total number of processes (GPUs) for training.") parser.add_argument( "--gpu_ranks", default=[], nargs='+', type=int, help="List of ranks of each process." " Each process has a unique integer rank whose value is in the interval [0, world_size), and runs in a single GPU." ) parser.add_argument("--master_ip", default="tcp://localhost:12345", type=str, help="IP-Port of master for training.") parser.add_argument("--backend", choices=["nccl", "gloo"], default="nccl", type=str, help="Distributed backend.") args = parser.parse_args() # Load hyper-parameters from config file. if args.config_path: load_hyperparam(args) ranks_num = len(args.gpu_ranks) if args.world_size > 1: # Multiprocessing distributed mode. assert torch.cuda.is_available(), "No available GPUs." assert ranks_num <= args.world_size, "Started processes exceed `world_size` upper limit." assert ranks_num <= torch.cuda.device_count( ), "Started processes exceeds the available GPUs." args.dist_train = True args.ranks_num = ranks_num print("Using distributed mode for training.") elif args.world_size == 1 and ranks_num == 1: # Single GPU mode. assert torch.cuda.is_available(), "No available GPUs." args.gpu_id = args.gpu_ranks[0] assert args.gpu_id < torch.cuda.device_count( ), "Invalid specified GPU device." args.dist_train = False args.single_gpu = True print("Using single GPU:%d for training." % args.gpu_id) else: # CPU mode. assert ranks_num == 0, "GPUs are specified, please check the arguments." args.dist_train = False args.single_gpu = False print("Using CPU mode for training.") trainer.train_and_validate(args)
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) # Path options. parser.add_argument("--dataset_path", type=str, default="dataset.pt", help="Path of the preprocessed dataset.") parser.add_argument("--vocab_path", default=None, type=str, help="Path of the vocabulary file.") parser.add_argument("--spm_model_path", default=None, type=str, help="Path of the sentence piece model.") parser.add_argument("--pretrained_model_path", type=str, default=None, help="Path of the pretrained model.") parser.add_argument("--output_model_path", type=str, required=True, help="Path of the output model.") parser.add_argument("--config_path", type=str, default="models/bert_base_config.json", help="Config file of model hyper-parameters.") # Training and saving options. parser.add_argument("--total_steps", type=int, default=100000, help="Total training steps.") parser.add_argument("--save_checkpoint_steps", type=int, default=10000, help="Specific steps to save model checkpoint.") parser.add_argument("--report_steps", type=int, default=100, help="Specific steps to print prompt.") parser.add_argument("--accumulation_steps", type=int, default=1, help="Specific steps to accumulate gradient.") parser.add_argument( "--batch_size", type=int, default=32, help= "Training batch size. The actual batch_size is [batch_size x world_size x accumulation_steps]." ) parser.add_argument("--instances_buffer_size", type=int, default=25600, help="The buffer size of instances in memory.") # Model options. parser.add_argument("--dropout", type=float, default=0.1, help="Dropout value.") parser.add_argument("--seed", type=int, default=7, help="Random seed.") parser.add_argument("--embedding", choices=["bert", "word", "gpt"], default="bert", help="Emebdding type.") parser.add_argument("--encoder", choices=["bert", "lstm", "gru", \ "cnn", "gatedcnn", "attn", "synt", \ "rcnn", "crnn", "gpt", "gpt2", "bilstm"], \ default="bert", help="Encoder type.") parser.add_argument("--bidirectional", action="store_true", help="Specific to recurrent model.") parser.add_argument("--target", choices=["bert", "lm", "cls", "mlm", "bilm", "albert"], default="bert", help="The training target of the pretraining model.") parser.add_argument("--tie_weights", action="store_true", help="Tie the word embedding and softmax weights.") parser.add_argument("--factorized_embedding_parameterization", action="store_true", help="Factorized embedding parameterization.") parser.add_argument("--has_lmtarget_bias", action="store_true", help="Add bias on output_layer for lm target.") parser.add_argument("--parameter_sharing", action="store_true", help="Parameter sharing.") # Masking options. parser.add_argument("--span_masking", action="store_true", help="Span masking.") parser.add_argument( "--span_geo_prob", type=float, default=0.2, help="Hyperparameter of geometric distribution for span masking.") parser.add_argument("--span_max_length", type=int, default=10, help="Max length for span masking.") # Optimizer options. parser.add_argument("--learning_rate", type=float, default=2e-5, help="Initial learning rate.") parser.add_argument("--warmup", type=float, default=0.1, help="Warm up value.") parser.add_argument("--beta1", type=float, default=0.9, help="Beta1 for Adam optimizer.") parser.add_argument("--beta2", type=float, default=0.999, help="Beta2 for Adam optimizer.") parser.add_argument( "--fp16", action='store_true', help= "Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit" ) parser.add_argument( "--fp16_opt_level", choices=["O0", "O1", "O2", "O3"], default='O1', help= "For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']." "See details at https://nvidia.github.io/apex/amp.html") # GPU options. parser.add_argument("--world_size", type=int, default=1, help="Total number of processes (GPUs) for training.") parser.add_argument( "--gpu_ranks", default=[], nargs='+', type=int, help="List of ranks of each process." " Each process has a unique integer rank whose value is in the interval [0, world_size), and runs in a single GPU." ) parser.add_argument("--master_ip", default="tcp://localhost:12345", type=str, help="IP-Port of master for training.") parser.add_argument("--backend", choices=["nccl", "gloo"], default="nccl", type=str, help="Distributed backend.") args = parser.parse_args() # Load hyper-parameters from config file. if args.config_path: load_hyperparam(args) ranks_num = len(args.gpu_ranks) if args.world_size > 1: # Multiprocessing distributed mode. assert torch.cuda.is_available(), "No available GPUs." assert ranks_num <= args.world_size, "Started processes exceed `world_size` upper limit." assert ranks_num <= torch.cuda.device_count( ), "Started processes exceeds the available GPUs." args.dist_train = True args.ranks_num = ranks_num print("Using distributed mode for training.") elif args.world_size == 1 and ranks_num == 1: # Single GPU mode. assert torch.cuda.is_available(), "No available GPUs." args.gpu_id = args.gpu_ranks[0] assert args.gpu_id < torch.cuda.device_count( ), "Invalid specified GPU device." args.dist_train = False args.single_gpu = True print("Using GPU %d for training." % args.gpu_id) else: # CPU mode. assert ranks_num == 0, "GPUs are specified, please check the arguments." args.dist_train = False args.single_gpu = False print("Using CPU mode for training.") trainer.train_and_validate(args)
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) # Path options. parser.add_argument("--dataset_path", type=str, default="dataset.pt", help="Path of the preprocessed dataset.") parser.add_argument("--pretrained_model_path", type=str, default=None, help="Path of the pretrained model.") parser.add_argument("--output_model_path", type=str, required=True, help="Path of the output model.") parser.add_argument("--config_path", type=str, default="models/bert/base_config.json", help="Config file of model hyper-parameters.") # Training and saving options. parser.add_argument("--total_steps", type=int, default=100000, help="Total training steps.") parser.add_argument("--save_checkpoint_steps", type=int, default=10000, help="Specific steps to save model checkpoint.") parser.add_argument("--report_steps", type=int, default=100, help="Specific steps to print prompt.") parser.add_argument("--accumulation_steps", type=int, default=1, help="Specific steps to accumulate gradient.") parser.add_argument( "--batch_size", type=int, default=32, help= "Training batch size. The actual batch_size is [batch_size x world_size x accumulation_steps]." ) parser.add_argument("--instances_buffer_size", type=int, default=25600, help="The buffer size of instances in memory.") parser.add_argument("--labels_num", type=int, required=False, help="Number of prediction labels.") parser.add_argument("--dropout", type=float, default=0.1, help="Dropout value.") parser.add_argument("--seed", type=int, default=7, help="Random seed.") # Preprocess options. tokenizer_opts(parser) tgt_tokenizer_opts(parser) # Model options. model_opts(parser) parser.add_argument("--data_processor", choices=[ "bert", "lm", "mlm", "bilm", "albert", "mt", "t5", "cls", "prefixlm", "gsg", "bart", "cls_mlm" ], default="bert", help="The data processor of the pretraining model.") parser.add_argument( "--deep_init", action="store_true", help="Scaling initialization of projection layers by a " "factor of 1/sqrt(2N). Necessary to large models.") # Masking options. parser.add_argument("--whole_word_masking", action="store_true", help="Whole word masking.") parser.add_argument("--span_masking", action="store_true", help="Span masking.") parser.add_argument( "--span_geo_prob", type=float, default=0.2, help="Hyperparameter of geometric distribution for span masking.") parser.add_argument("--span_max_length", type=int, default=10, help="Max length for span masking.") # Optimizer options. optimization_opts(parser) # GPU options. parser.add_argument("--world_size", type=int, default=1, help="Total number of processes (GPUs) for training.") parser.add_argument( "--gpu_ranks", default=[], nargs='+', type=int, help="List of ranks of each process." " Each process has a unique integer rank whose value is in the interval [0, world_size), and runs in a single GPU." ) parser.add_argument("--master_ip", default="tcp://localhost:12345", type=str, help="IP-Port of master for training.") parser.add_argument("--backend", choices=["nccl", "gloo"], default="nccl", type=str, help="Distributed backend.") # Deepspeed options. deepspeed_opts(parser) # Log options. log_opts(parser) args = parser.parse_args() if "cls" in args.target: assert args.labels_num is not None, "Cls target needs the denotation of the number of labels." # Load hyper-parameters from config file. if args.config_path: args = load_hyperparam(args) ranks_num = len(args.gpu_ranks) if args.deepspeed: if args.world_size > 1: args.dist_train = True else: args.dist_train = False else: if args.world_size > 1: # Multiprocessing distributed mode. assert torch.cuda.is_available(), "No available GPUs." assert ranks_num <= args.world_size, "Started processes exceed `world_size` upper limit." assert ranks_num <= torch.cuda.device_count( ), "Started processes exceeds the available GPUs." args.dist_train = True args.ranks_num = ranks_num print("Using distributed mode for training.") elif args.world_size == 1 and ranks_num == 1: # Single GPU mode. assert torch.cuda.is_available(), "No available GPUs." args.gpu_id = args.gpu_ranks[0] assert args.gpu_id < torch.cuda.device_count( ), "Invalid specified GPU device." args.dist_train = False args.single_gpu = True print("Using GPU %d for training." % args.gpu_id) else: # CPU mode. assert ranks_num == 0, "GPUs are specified, please check the arguments." args.dist_train = False args.single_gpu = False print("Using CPU mode for training.") trainer.train_and_validate(args)