def main():
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# Path options.
parser.add_argument("--dataset_path", type=str, default="dataset.pt",
help="Path of the preprocessed dataset.")
parser.add_argument("--vocab_path", default=None, type=str,
help="Path of the vocabulary file.")
parser.add_argument("--spm_model_path", default=None, type=str,
help="Path of the sentence piece model.")
parser.add_argument("--tgt_vocab_path", default=None, type=str,
help="Path of the target vocabulary file.")
parser.add_argument("--tgt_spm_model_path", default=None, type=str,
help="Path of the target sentence piece model.")
parser.add_argument("--pretrained_model_path", type=str, default=None,
help="Path of the pretrained model.")
parser.add_argument("--output_model_path", type=str, required=True,
help="Path of the output model.")
parser.add_argument("--config_path", type=str, default="models/bert/base_config.json",
help="Config file of model hyper-parameters.")
# Training and saving options.
parser.add_argument("--total_steps", type=int, default=100000,
help="Total training steps.")
parser.add_argument("--save_checkpoint_steps", type=int, default=10000,
help="Specific steps to save model checkpoint.")
parser.add_argument("--report_steps", type=int, default=100,
help="Specific steps to print prompt.")
parser.add_argument("--accumulation_steps", type=int, default=1,
help="Specific steps to accumulate gradient.")
parser.add_argument("--batch_size", type=int, default=32,
help="Training batch size. The actual batch_size is [batch_size x world_size x accumulation_steps].")
parser.add_argument("--instances_buffer_size", type=int, default=25600,
help="The buffer size of instances in memory.")
parser.add_argument("--labels_num", type=int, required=False,
help="Number of prediction labels.")
parser.add_argument("--dropout", type=float, default=0.1, help="Dropout value.")
parser.add_argument("--seed", type=int, default=7, help="Random seed.")
# Preprocess options.
parser.add_argument("--tokenizer", choices=["bert", "char", "space", "xlmroberta"], default="bert",
help="Specify the tokenizer."
"Original Google BERT uses bert tokenizer."
"Char tokenizer segments sentences into characters."
"Space tokenizer segments sentences into words according to space."
"Original XLM-RoBERTa uses xlmroberta tokenizer."
)
parser.add_argument("--tgt_tokenizer", choices=["bert", "char", "space", "xlmroberta"], default="bert",
help="Specify the tokenizer for target side.")
# Model options.
model_opts(parser)
parser.add_argument("--tgt_embedding", choices=["word", "word_pos", "word_pos_seg", "word_sinusoidalpos"], default="word_pos_seg",
help="Target embedding type.")
parser.add_argument("--decoder", choices=["transformer"], default="transformer", help="Decoder type.")
parser.add_argument("--pooling", choices=["mean", "max", "first", "last"], default="first",
help="Pooling type.")
parser.add_argument("--target", choices=["bert", "lm", "mlm", "bilm", "albert", "seq2seq", "t5", "cls", "prefixlm", "gsg", "bart"], default="bert",
help="The training target of the pretraining model.")
parser.add_argument("--tie_weights", action="store_true",
help="Tie the word embedding and softmax weights.")
parser.add_argument("--has_lmtarget_bias", action="store_true",
help="Add bias on output_layer for lm target.")
parser.add_argument("--deep_init", action="store_true",
help="initialize bert model similar to gpt2 model."
"scales initialization of projection layers by a "
"factor of 1/sqrt(2N). Necessary to train bert "
"models larger than BERT-Large.")
# Masking options.
parser.add_argument("--whole_word_masking", action="store_true", help="Whole word masking.")
parser.add_argument("--span_masking", action="store_true", help="Span masking.")
parser.add_argument("--span_geo_prob", type=float, default=0.2,
help="Hyperparameter of geometric distribution for span masking.")
parser.add_argument("--span_max_length", type=int, default=10,
help="Max length for span masking.")
# Optimizer options.
optimization_opts(parser)
# GPU options.
parser.add_argument("--world_size", type=int, default=1, help="Total number of processes (GPUs) for training.")
parser.add_argument("--gpu_ranks", default=[], nargs='+', type=int, help="List of ranks of each process."
" Each process has a unique integer rank whose value is in the interval [0, world_size), and runs in a single GPU.")
parser.add_argument("--master_ip", default="tcp://localhost:12345", type=str, help="IP-Port of master for training.")
parser.add_argument("--backend", choices=["nccl", "gloo"], default="nccl", type=str, help="Distributed backend.")
args = parser.parse_args()
if args.target == "cls":
assert args.labels_num is not None, "Cls target needs the denotation of the number of labels."
# Load hyper-parameters from config file.
if args.config_path:
load_hyperparam(args)
ranks_num = len(args.gpu_ranks)
if args.world_size > 1:
# Multiprocessing distributed mode.
assert torch.cuda.is_available(), "No available GPUs."
assert ranks_num <= args.world_size, "Started processes exceed `world_size` upper limit."
assert ranks_num <= torch.cuda.device_count(), "Started processes exceeds the available GPUs."
args.dist_train = True
args.ranks_num = ranks_num
print("Using distributed mode for training.")
elif args.world_size == 1 and ranks_num == 1:
# Single GPU mode.
assert torch.cuda.is_available(), "No available GPUs."
args.gpu_id = args.gpu_ranks[0]
assert args.gpu_id < torch.cuda.device_count(), "Invalid specified GPU device."
args.dist_train = False
args.single_gpu = True
print("Using GPU %d for training." % args.gpu_id)
else:
# CPU mode.
assert ranks_num == 0, "GPUs are specified, please check the arguments."
args.dist_train = False
args.single_gpu = False
print("Using CPU mode for training.")
trainer.train_and_validate(args)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# Path options.
parser.add_argument("--dataset_path",
type=str,
default="dataset.pt",
help="Path of the preprocessed dataset.")
parser.add_argument("--vocab_path",
type=str,
required=True,
help="Path of the vocabulary file.")
parser.add_argument("--pretrained_model_path",
type=str,
default=None,
help="Path of the pretrained model.")
parser.add_argument("--output_model_path",
type=str,
required=True,
help="Path of the output model.")
parser.add_argument("--config_path",
type=str,
default=None,
help="Config file of model hyper-parameters.")
# Training and saving options.
parser.add_argument("--total_steps",
type=int,
default=100000,
help="Total training steps.")
parser.add_argument("--save_checkpoint_steps",
type=int,
default=10000,
help="Specific steps to save model checkpoint.")
parser.add_argument("--report_steps",
type=int,
default=100,
help="Specific steps to print prompt.")
parser.add_argument("--accumulation_steps",
type=int,
default=1,
help="Specific steps to accumulate gradient.")
parser.add_argument(
"--batch_size",
type=int,
default=32,
help=
"Training batch size. The actual batch_size is [batch_size x world_size x accumulation_steps]."
)
parser.add_argument("--instances_buffer_size",
type=int,
default=25600,
help="The buffer size of instances in memory.")
# Model options.
parser.add_argument("--emb_size",
type=int,
default=768,
help="Embedding dimension.")
parser.add_argument("--hidden_size",
type=int,
default=768,
help="Hidden state dimension.")
parser.add_argument("--feedforward_size",
type=int,
default=3072,
help="Feed forward layer dimension.")
parser.add_argument("--kernel_size",
type=int,
default=3,
help="Kernel size for CNN.")
parser.add_argument("--block_size",
type=int,
default=2,
help="Block size for CNN.")
parser.add_argument("--heads_num",
type=int,
default=12,
help="The number of heads in multi-head attention.")
parser.add_argument("--layers_num",
type=int,
default=12,
help="The number of encoder layers.")
parser.add_argument("--dropout",
type=float,
default=0.1,
help="Dropout value.")
parser.add_argument("--seed", type=int, default=7, help="Random seed.")
parser.add_argument("--encoder", choices=["bert", "lstm", "gru", \
"cnn", "gatedcnn", "attn", \
"rcnn", "crnn", "gpt", "bilstm"], \
default="bert", help="Encoder type.")
parser.add_argument("--bidirectional",
action="store_true",
help="Specific to recurrent model.")
parser.add_argument("--target",
choices=["bert", "lm", "cls", "mlm", "bilm"],
default="bert",
help="The training target of the pretraining model.")
parser.add_argument("--labels_num",
type=int,
default=2,
help="Specific to classification target.")
# Optimizer options.
parser.add_argument("--learning_rate",
type=float,
default=2e-5,
help="Initial learning rate.")
parser.add_argument("--warmup",
type=float,
default=0.1,
help="Warm up value.")
# Subword options.
parser.add_argument("--subword_type",
choices=["none", "char"],
default="none",
help="Subword feature type.")
parser.add_argument("--sub_vocab_path",
type=str,
default="models/sub_vocab.txt",
help="Path of the subword vocabulary file.")
parser.add_argument("--subencoder",
choices=["avg", "lstm", "gru", "cnn"],
default="avg",
help="Subencoder type.")
parser.add_argument("--sub_layers_num",
type=int,
default=2,
help="The number of subencoder layers.")
# GPU options.
parser.add_argument("--world_size",
type=int,
default=1,
help="Total number of processes (GPUs) for training.")
parser.add_argument(
"--gpu_ranks",
default=[],
nargs='+',
type=int,
help="List of ranks of each process."
" Each process has a unique integer rank whose value is in the interval [0, world_size), and runs in a single GPU."
)
parser.add_argument("--master_ip",
default="tcp://localhost:12345",
type=str,
help="IP-Port of master for training.")
parser.add_argument("--backend",
choices=["nccl", "gloo"],
default="nccl",
type=str,
help="Distributed backend.")
args = parser.parse_args()
# Load hyper-parameters from config file.
if args.config_path:
load_hyperparam(args)
ranks_num = len(args.gpu_ranks)
if args.world_size > 1:
# Multiprocessing distributed mode.
assert torch.cuda.is_available(), "No available GPUs."
assert ranks_num <= args.world_size, "Started processes exceed `world_size` upper limit."
assert ranks_num <= torch.cuda.device_count(
), "Started processes exceeds the available GPUs."
args.dist_train = True
args.ranks_num = ranks_num
print("Using distributed mode for training.")
elif args.world_size == 1 and ranks_num == 1:
# Single GPU mode.
assert torch.cuda.is_available(), "No available GPUs."
args.gpu_id = args.gpu_ranks[0]
assert args.gpu_id < torch.cuda.device_count(
), "Invalid specified GPU device."
args.dist_train = False
args.single_gpu = True
print("Using single GPU:%d for training." % args.gpu_id)
else:
# CPU mode.
assert ranks_num == 0, "GPUs are specified, please check the arguments."
args.dist_train = False
args.single_gpu = False
print("Using CPU mode for training.")
trainer.train_and_validate(args)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# Path options.
parser.add_argument("--dataset_path",
type=str,
default="dataset.pt",
help="Path of the preprocessed dataset.")
parser.add_argument("--vocab_path",
default=None,
type=str,
help="Path of the vocabulary file.")
parser.add_argument("--spm_model_path",
default=None,
type=str,
help="Path of the sentence piece model.")
parser.add_argument("--pretrained_model_path",
type=str,
default=None,
help="Path of the pretrained model.")
parser.add_argument("--output_model_path",
type=str,
required=True,
help="Path of the output model.")
parser.add_argument("--config_path",
type=str,
default="models/bert_base_config.json",
help="Config file of model hyper-parameters.")
# Training and saving options.
parser.add_argument("--total_steps",
type=int,
default=100000,
help="Total training steps.")
parser.add_argument("--save_checkpoint_steps",
type=int,
default=10000,
help="Specific steps to save model checkpoint.")
parser.add_argument("--report_steps",
type=int,
default=100,
help="Specific steps to print prompt.")
parser.add_argument("--accumulation_steps",
type=int,
default=1,
help="Specific steps to accumulate gradient.")
parser.add_argument(
"--batch_size",
type=int,
default=32,
help=
"Training batch size. The actual batch_size is [batch_size x world_size x accumulation_steps]."
)
parser.add_argument("--instances_buffer_size",
type=int,
default=25600,
help="The buffer size of instances in memory.")
# Model options.
parser.add_argument("--dropout",
type=float,
default=0.1,
help="Dropout value.")
parser.add_argument("--seed", type=int, default=7, help="Random seed.")
parser.add_argument("--embedding",
choices=["bert", "word", "gpt"],
default="bert",
help="Emebdding type.")
parser.add_argument("--encoder", choices=["bert", "lstm", "gru", \
"cnn", "gatedcnn", "attn", "synt", \
"rcnn", "crnn", "gpt", "gpt2", "bilstm"], \
default="bert", help="Encoder type.")
parser.add_argument("--bidirectional",
action="store_true",
help="Specific to recurrent model.")
parser.add_argument("--target",
choices=["bert", "lm", "cls", "mlm", "bilm", "albert"],
default="bert",
help="The training target of the pretraining model.")
parser.add_argument("--tie_weights",
action="store_true",
help="Tie the word embedding and softmax weights.")
parser.add_argument("--factorized_embedding_parameterization",
action="store_true",
help="Factorized embedding parameterization.")
parser.add_argument("--has_lmtarget_bias",
action="store_true",
help="Add bias on output_layer for lm target.")
parser.add_argument("--parameter_sharing",
action="store_true",
help="Parameter sharing.")
# Masking options.
parser.add_argument("--span_masking",
action="store_true",
help="Span masking.")
parser.add_argument(
"--span_geo_prob",
type=float,
default=0.2,
help="Hyperparameter of geometric distribution for span masking.")
parser.add_argument("--span_max_length",
type=int,
default=10,
help="Max length for span masking.")
# Optimizer options.
parser.add_argument("--learning_rate",
type=float,
default=2e-5,
help="Initial learning rate.")
parser.add_argument("--warmup",
type=float,
default=0.1,
help="Warm up value.")
parser.add_argument("--beta1",
type=float,
default=0.9,
help="Beta1 for Adam optimizer.")
parser.add_argument("--beta2",
type=float,
default=0.999,
help="Beta2 for Adam optimizer.")
parser.add_argument(
"--fp16",
action='store_true',
help=
"Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"
)
parser.add_argument(
"--fp16_opt_level",
choices=["O0", "O1", "O2", "O3"],
default='O1',
help=
"For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
"See details at https://nvidia.github.io/apex/amp.html")
# GPU options.
parser.add_argument("--world_size",
type=int,
default=1,
help="Total number of processes (GPUs) for training.")
parser.add_argument(
"--gpu_ranks",
default=[],
nargs='+',
type=int,
help="List of ranks of each process."
" Each process has a unique integer rank whose value is in the interval [0, world_size), and runs in a single GPU."
)
parser.add_argument("--master_ip",
default="tcp://localhost:12345",
type=str,
help="IP-Port of master for training.")
parser.add_argument("--backend",
choices=["nccl", "gloo"],
default="nccl",
type=str,
help="Distributed backend.")
args = parser.parse_args()
# Load hyper-parameters from config file.
if args.config_path:
load_hyperparam(args)
ranks_num = len(args.gpu_ranks)
if args.world_size > 1:
# Multiprocessing distributed mode.
assert torch.cuda.is_available(), "No available GPUs."
assert ranks_num <= args.world_size, "Started processes exceed `world_size` upper limit."
assert ranks_num <= torch.cuda.device_count(
), "Started processes exceeds the available GPUs."
args.dist_train = True
args.ranks_num = ranks_num
print("Using distributed mode for training.")
elif args.world_size == 1 and ranks_num == 1:
# Single GPU mode.
assert torch.cuda.is_available(), "No available GPUs."
args.gpu_id = args.gpu_ranks[0]
assert args.gpu_id < torch.cuda.device_count(
), "Invalid specified GPU device."
args.dist_train = False
args.single_gpu = True
print("Using GPU %d for training." % args.gpu_id)
else:
# CPU mode.
assert ranks_num == 0, "GPUs are specified, please check the arguments."
args.dist_train = False
args.single_gpu = False
print("Using CPU mode for training.")
trainer.train_and_validate(args)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# Path options.
parser.add_argument("--dataset_path",
type=str,
default="dataset.pt",
help="Path of the preprocessed dataset.")
parser.add_argument("--pretrained_model_path",
type=str,
default=None,
help="Path of the pretrained model.")
parser.add_argument("--output_model_path",
type=str,
required=True,
help="Path of the output model.")
parser.add_argument("--config_path",
type=str,
default="models/bert/base_config.json",
help="Config file of model hyper-parameters.")
# Training and saving options.
parser.add_argument("--total_steps",
type=int,
default=100000,
help="Total training steps.")
parser.add_argument("--save_checkpoint_steps",
type=int,
default=10000,
help="Specific steps to save model checkpoint.")
parser.add_argument("--report_steps",
type=int,
default=100,
help="Specific steps to print prompt.")
parser.add_argument("--accumulation_steps",
type=int,
default=1,
help="Specific steps to accumulate gradient.")
parser.add_argument(
"--batch_size",
type=int,
default=32,
help=
"Training batch size. The actual batch_size is [batch_size x world_size x accumulation_steps]."
)
parser.add_argument("--instances_buffer_size",
type=int,
default=25600,
help="The buffer size of instances in memory.")
parser.add_argument("--labels_num",
type=int,
required=False,
help="Number of prediction labels.")
parser.add_argument("--dropout",
type=float,
default=0.1,
help="Dropout value.")
parser.add_argument("--seed", type=int, default=7, help="Random seed.")
# Preprocess options.
tokenizer_opts(parser)
tgt_tokenizer_opts(parser)
# Model options.
model_opts(parser)
parser.add_argument("--data_processor",
choices=[
"bert", "lm", "mlm", "bilm", "albert", "mt", "t5",
"cls", "prefixlm", "gsg", "bart", "cls_mlm"
],
default="bert",
help="The data processor of the pretraining model.")
parser.add_argument(
"--deep_init",
action="store_true",
help="Scaling initialization of projection layers by a "
"factor of 1/sqrt(2N). Necessary to large models.")
# Masking options.
parser.add_argument("--whole_word_masking",
action="store_true",
help="Whole word masking.")
parser.add_argument("--span_masking",
action="store_true",
help="Span masking.")
parser.add_argument(
"--span_geo_prob",
type=float,
default=0.2,
help="Hyperparameter of geometric distribution for span masking.")
parser.add_argument("--span_max_length",
type=int,
default=10,
help="Max length for span masking.")
# Optimizer options.
optimization_opts(parser)
# GPU options.
parser.add_argument("--world_size",
type=int,
default=1,
help="Total number of processes (GPUs) for training.")
parser.add_argument(
"--gpu_ranks",
default=[],
nargs='+',
type=int,
help="List of ranks of each process."
" Each process has a unique integer rank whose value is in the interval [0, world_size), and runs in a single GPU."
)
parser.add_argument("--master_ip",
default="tcp://localhost:12345",
type=str,
help="IP-Port of master for training.")
parser.add_argument("--backend",
choices=["nccl", "gloo"],
default="nccl",
type=str,
help="Distributed backend.")
# Deepspeed options.
deepspeed_opts(parser)
# Log options.
log_opts(parser)
args = parser.parse_args()
if "cls" in args.target:
assert args.labels_num is not None, "Cls target needs the denotation of the number of labels."
# Load hyper-parameters from config file.
if args.config_path:
args = load_hyperparam(args)
ranks_num = len(args.gpu_ranks)
if args.deepspeed:
if args.world_size > 1:
args.dist_train = True
else:
args.dist_train = False
else:
if args.world_size > 1:
# Multiprocessing distributed mode.
assert torch.cuda.is_available(), "No available GPUs."
assert ranks_num <= args.world_size, "Started processes exceed `world_size` upper limit."
assert ranks_num <= torch.cuda.device_count(
), "Started processes exceeds the available GPUs."
args.dist_train = True
args.ranks_num = ranks_num
print("Using distributed mode for training.")
elif args.world_size == 1 and ranks_num == 1:
# Single GPU mode.
assert torch.cuda.is_available(), "No available GPUs."
args.gpu_id = args.gpu_ranks[0]
assert args.gpu_id < torch.cuda.device_count(
), "Invalid specified GPU device."
args.dist_train = False
args.single_gpu = True
print("Using GPU %d for training." % args.gpu_id)
else:
# CPU mode.
assert ranks_num == 0, "GPUs are specified, please check the arguments."
args.dist_train = False
args.single_gpu = False
print("Using CPU mode for training.")
trainer.train_and_validate(args)
