def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
finetune_opts(parser)
parser.add_argument("--train_answer_path",
type=str,
required=True,
help="Path of the answers for trainset.")
parser.add_argument("--dev_answer_path",
type=str,
required=True,
help="Path of the answers for devset.")
parser.add_argument(
"--max_choices_num",
default=10,
type=int,
help=
"The maximum number of cadicate answer, shorter than this will be padded."
)
args = parser.parse_args()
args.labels_num = args.max_choices_num
if args.output_model_path == None:
args.output_model_path = "./models/chid_model.bin"
# Load the hyperparameters from the config file.
args = load_hyperparam(args)
set_seed(args.seed)
# Build tokenizer.
args.tokenizer = CharTokenizer(args)
# Build multiple choice model.
model = MultipleChoice(args)
# Load or initialize parameters.
load_or_initialize_parameters(args, model)
args.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = model.to(args.device)
# Training phase.
trainset = read_dataset(args, args.train_path, args.train_answer_path)
random.shuffle(trainset)
instances_num = len(trainset)
batch_size = args.batch_size
src = torch.LongTensor([example[0] for example in trainset])
tgt = torch.LongTensor([example[1] for example in trainset])
seg = torch.LongTensor([example[2] for example in trainset])
args.train_steps = int(instances_num * args.epochs_num / batch_size) + 1
print("Batch size: ", batch_size)
print("The number of training instances:", instances_num)
optimizer, scheduler = build_optimizer(args, model)
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
model, optimizer = amp.initialize(model,
optimizer,
opt_level=args.fp16_opt_level)
args.amp = amp
if torch.cuda.device_count() > 1:
print("{} GPUs are available. Let's use them.".format(
torch.cuda.device_count()))
model = torch.nn.DataParallel(model)
args.model = model
total_loss, result, best_result = 0., 0., 0.
print("Start training.")
for epoch in range(1, args.epochs_num + 1):
model.train()
for i, (src_batch, tgt_batch, seg_batch,
_) in enumerate(batch_loader(batch_size, src, tgt, seg)):
loss = train_model(args, model, optimizer, scheduler, src_batch,
tgt_batch, seg_batch)
total_loss += loss.item()
if (i + 1) % args.report_steps == 0:
print("Epoch id: {}, Training steps: {}, Avg loss: {:.3f}".
format(epoch, i + 1, total_loss / args.report_steps))
total_loss = 0.
result = evaluate(
args, read_dataset(args, args.dev_path, args.dev_answer_path))
if result[0] > best_result:
best_result = result[0]
save_model(model, args.output_model_path)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
finetune_opts(parser)
parser.add_argument(
"--doc_stride",
default=128,
type=int,
help=
"When splitting up a long document into chunks, how much stride to take between chunks."
)
args = parser.parse_args()
# Load the hyperparameters from the config file.
args = load_hyperparam(args)
set_seed(args.seed)
# Build tokenizer.
args.tokenizer = CharTokenizer(args)
# Build machine reading comprehension model.
model = MachineReadingComprehension(args)
# Load or initialize parameters.
load_or_initialize_parameters(args, model)
args.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = model.to(args.device)
# Build tokenizer.
args.tokenizer = CharTokenizer(args)
# Training phase.
batch_size = args.batch_size
print("Batch size: ", batch_size)
trainset, _ = read_dataset(args, args.train_path)
random.shuffle(trainset)
instances_num = len(trainset)
src = torch.LongTensor([sample[0] for sample in trainset])
seg = torch.LongTensor([sample[1] for sample in trainset])
start_position = torch.LongTensor([sample[2] for sample in trainset])
end_position = torch.LongTensor([sample[3] for sample in trainset])
args.train_steps = int(instances_num * args.epochs_num / batch_size) + 1
print("The number of training instances:", instances_num)
optimizer, scheduler = build_optimizer(args, model)
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
model, optimizer = amp.initialize(model,
optimizer,
opt_level=args.fp16_opt_level)
if torch.cuda.device_count() > 1:
print("{} GPUs are available. Let's use them.".format(
torch.cuda.device_count()))
model = torch.nn.DataParallel(model)
args.model = model
total_loss = 0.0
result = 0.0
best_result = 0.0
print("Start training.")
for epoch in range(1, args.epochs_num + 1):
model.train()
for i, (src_batch, seg_batch, start_position_batch,
end_position_batch) in enumerate(
batch_loader(batch_size, src, seg, start_position,
end_position)):
loss = train(args, model, optimizer, scheduler, src_batch,
seg_batch, start_position_batch, end_position_batch)
total_loss += loss.item()
if (i + 1) % args.report_steps == 0:
print("Epoch id: {}, Training steps: {}, Avg loss: {:.3f}".
format(epoch, i + 1, total_loss / args.report_steps))
total_loss = 0.0
result = evaluate(args, *read_dataset(args, args.dev_path))
if result > best_result:
best_result = result
save_model(model, args.output_model_path)
# Evaluation phase.
if args.test_path is not None:
print("Test set evaluation.")
if torch.cuda.device_count() > 1:
model.module.load_state_dict(torch.load(args.output_model_path))
else:
model.load_state_dict(torch.load(args.output_model_path))
evaluate(args, *read_dataset(args, args.test_path))
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
finetune_opts(parser)
parser.add_argument(
"--max_choices_num",
default=4,
type=int,
help=
"The maximum number of cadicate answer, shorter than this will be padded."
)
parser.add_argument(
"--tokenizer",
choices=["bert", "char", "space"],
default="bert",
help="Specify the tokenizer."
"Original Google BERT uses bert tokenizer on Chinese corpus."
"Char tokenizer segments sentences into characters."
"Space tokenizer segments sentences into words according to space.")
args = parser.parse_args()
args.labels_num = args.max_choices_num
# Load the hyperparameters from the config file.
args = load_hyperparam(args)
set_seed(args.seed)
# Build tokenizer.
args.tokenizer = str2tokenizer[args.tokenizer](args)
# Build multiple choice model.
model = MultipleChoice(args)
# Load or initialize parameters.
load_or_initialize_parameters(args, model)
args.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = model.to(args.device)
# Training phase.
trainset = read_dataset(args, args.train_path)
random.shuffle(trainset)
instances_num = len(trainset)
batch_size = args.batch_size
src = torch.LongTensor([example[0] for example in trainset])
tgt = torch.LongTensor([example[1] for example in trainset])
seg = torch.LongTensor([example[2] for example in trainset])
args.train_steps = int(instances_num * args.epochs_num / batch_size) + 1
print("Batch size: ", batch_size)
print("The number of training instances:", instances_num)
optimizer, scheduler = build_optimizer(args, model)
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
model, optimizer = amp.initialize(model,
optimizer,
opt_level=args.fp16_opt_level)
args.amp = amp
if torch.cuda.device_count() > 1:
print("{} GPUs are available. Let's use them.".format(
torch.cuda.device_count()))
model = torch.nn.DataParallel(model)
args.model = model
total_loss, result, best_result = 0.0, 0.0, 0.0
print("Start training.")
for epoch in range(1, args.epochs_num + 1):
model.train()
for i, (src_batch, tgt_batch, seg_batch,
_) in enumerate(batch_loader(batch_size, src, tgt, seg)):
loss = train_model(args, model, optimizer, scheduler, src_batch,
tgt_batch, seg_batch)
total_loss += loss.item()
if (i + 1) % args.report_steps == 0:
print("Epoch id: {}, Training steps: {}, Avg loss: {:.3f}".
format(epoch, i + 1, total_loss / args.report_steps))
total_loss = 0.0
result = evaluate(args, read_dataset(args, args.dev_path))
if result[0] > best_result:
best_result = result[0]
save_model(model, args.output_model_path)
# Evaluation phase.
if args.test_path is not None:
print("Test set evaluation.")
if torch.cuda.device_count() > 1:
model.module.load_state_dict(torch.load(args.output_model_path))
else:
model.load_state_dict(torch.load(args.output_model_path))
evaluate(args, read_dataset(args, args.test_path))
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# Path options.
parser.add_argument("--pretrained_model_path",
default=None,
type=str,
help="Path of the pretrained model.")
parser.add_argument("--dataset_path_list",
default=[],
nargs='+',
type=str,
help="Dataset path list.")
parser.add_argument("--output_model_path",
default="models/multitask_classifier_model.bin",
type=str,
help="Path of the output model.")
parser.add_argument("--vocab_path",
default=None,
type=str,
help="Path of the vocabulary file.")
parser.add_argument("--spm_model_path",
default=None,
type=str,
help="Path of the sentence piece model.")
parser.add_argument("--config_path",
default="models/bert_base_config.json",
type=str,
help="Path of the config file.")
# Model options.
model_opts(parser)
parser.add_argument("--pooling",
choices=["mean", "max", "first", "last"],
default="first",
help="Pooling type.")
# Tokenizer options.
parser.add_argument(
"--tokenizer",
choices=["bert", "char", "space"],
default="bert",
help="Specify the tokenizer."
"Original Google BERT uses bert tokenizer on Chinese corpus."
"Char tokenizer segments sentences into characters."
"Space tokenizer segments sentences into words according to space.")
# Optimizer options.
optimization_opts(parser)
# Training options.
training_opts(parser)
args = parser.parse_args()
args.soft_targets = False
# Load the hyperparameters from the config file.
args = load_hyperparam(args)
set_seed(args.seed)
# Count the number of labels.
args.labels_num_list = [
count_labels_num(os.path.join(path, "train.tsv"))
for path in args.dataset_path_list
]
args.datasets_num = len(args.dataset_path_list)
# Build tokenizer.
args.tokenizer = str2tokenizer[args.tokenizer](args)
# Build multi-task classification model.
model = MultitaskClassifier(args)
# Load or initialize parameters.
load_or_initialize_parameters(args, model)
args.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = model.to(args.device)
args.model = model
# Training phase.
dataset_list = [
read_dataset(args, os.path.join(path, "train.tsv"))
for path in args.dataset_path_list
]
packed_dataset_list = [
pack_dataset(dataset, i, args.batch_size)
for i, dataset in enumerate(dataset_list)
]
packed_dataset_all = []
for packed_dataset in packed_dataset_list:
packed_dataset_all += packed_dataset
random.shuffle(packed_dataset_all)
instances_num = sum([len(dataset) for dataset in dataset_list])
batch_size = args.batch_size
args.train_steps = int(instances_num * args.epochs_num / batch_size) + 1
print("Batch size: ", batch_size)
print("The number of training instances:", instances_num)
optimizer, scheduler = build_optimizer(args, model)
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
model, optimizer = amp.initialize(model,
optimizer,
opt_level=args.fp16_opt_level)
args.amp = amp
if torch.cuda.device_count() > 1:
print("{} GPUs are available. Let's use them.".format(
torch.cuda.device_count()))
model = torch.nn.DataParallel(model)
total_loss, result, best_result = 0.0, 0.0, 0.0
print("Start training.")
for epoch in range(1, args.epochs_num + 1):
model.train()
for i, (dataset_id, src_batch, tgt_batch,
seg_batch) in enumerate(packed_dataset_all):
if hasattr(model, "module"):
model.module.change_dataset(dataset_id)
else:
model.change_dataset(dataset_id)
loss = train_model(args, model, optimizer, scheduler, src_batch,
tgt_batch, seg_batch, None)
total_loss += loss.item()
if (i + 1) % args.report_steps == 0:
print("Epoch id: {}, Training steps: {}, Avg loss: {:.3f}".
format(epoch, i + 1, total_loss / args.report_steps))
total_loss = 0.0
for dataset_id, path in enumerate(args.dataset_path_list):
args.labels_num = args.labels_num_list[dataset_id]
if hasattr(model, "module"):
model.module.change_dataset(dataset_id)
else:
model.change_dataset(dataset_id)
result = evaluate(
args, read_dataset(args, os.path.join(path, "dev.tsv")))
save_model(model, args.output_model_path)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# Path options.
parser.add_argument("--pretrained_model_path",
default=None,
type=str,
help="Path of the pretrained model.")
parser.add_argument("--output_model_path",
default="./models/cmrc_model.bin",
type=str,
help="Path of the output model.")
parser.add_argument("--vocab_path",
default=None,
type=str,
help="Path of the vocabulary file.")
parser.add_argument("--spm_model_path",
default=None,
type=str,
help="Path of the sentence piece model.")
parser.add_argument("--train_path",
type=str,
required=True,
help="Path of the trainset.")
parser.add_argument("--dev_path",
type=str,
required=True,
help="Path of the devset.")
parser.add_argument("--test_path", type=str, help="Path of the testset.")
parser.add_argument("--config_path",
default="./models/bert_base_config.json",
type=str,
help="Path of the config file.")
# Model options.
parser.add_argument("--batch_size",
type=int,
default=64,
help="Batch size.")
parser.add_argument("--seq_length",
type=int,
default=100,
help="Sequence length.")
parser.add_argument(
"--doc_stride",
default=128,
type=int,
help=
"When splitting up a long document into chunks, how much stride to take between chunks."
)
parser.add_argument("--embedding",
choices=["bert", "word"],
default="bert",
help="Emebdding type.")
parser.add_argument("--encoder", choices=["bert", "lstm", "gru", \
"cnn", "gatedcnn", "attn", "synt", \
"rcnn", "crnn", "gpt", "bilstm"], \
default="bert", help="Encoder type.")
parser.add_argument("--bidirectional",
action="store_true",
help="Specific to recurrent model.")
parser.add_argument("--factorized_embedding_parameterization",
action="store_true",
help="Factorized embedding parameterization.")
parser.add_argument("--parameter_sharing",
action="store_true",
help="Parameter sharing.")
# Optimizer options.
parser.add_argument("--learning_rate",
type=float,
default=3e-5,
help="Learning rate.")
parser.add_argument("--warmup",
type=float,
default=0.1,
help="Warm up value.")
parser.add_argument(
"--fp16",
action='store_true',
help=
"Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"
)
parser.add_argument(
"--fp16_opt_level",
choices=["O0", "O1", "O2", "O3"],
default='O1',
help=
"For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
"See details at https://nvidia.github.io/apex/amp.html")
# Training options.
parser.add_argument("--dropout", type=float, default=0.5, help="Dropout.")
parser.add_argument("--epochs_num",
type=int,
default=3,
help="Number of epochs.")
parser.add_argument("--report_steps",
type=int,
default=100,
help="Specific steps to print prompt.")
parser.add_argument("--seed", type=int, default=7, help="Random seed.")
args = parser.parse_args()
# Load the hyperparameters from the config file.
args = load_hyperparam(args)
set_seed(args.seed)
# Build tokenizer.
args.tokenizer = CharTokenizer(args)
# Build machine reading comprehension model.
model = MachineReadingComprehension(args)
# Load or initialize parameters.
load_or_initialize_parameters(args, model)
args.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = model.to(args.device)
# Build tokenizer.
args.tokenizer = CharTokenizer(args)
# Training phase.
batch_size = args.batch_size
print("Batch size: ", batch_size)
trainset, _ = read_dataset(args, args.train_path)
random.shuffle(trainset)
instances_num = len(trainset)
src = torch.LongTensor([sample[0] for sample in trainset])
seg = torch.LongTensor([sample[1] for sample in trainset])
start_position = torch.LongTensor([sample[2] for sample in trainset])
end_position = torch.LongTensor([sample[3] for sample in trainset])
args.train_steps = int(instances_num * args.epochs_num / batch_size) + 1
print("The number of training instances:", instances_num)
optimizer, scheduler = build_optimizer(args, model)
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
model, optimizer = amp.initialize(model,
optimizer,
opt_level=args.fp16_opt_level)
if torch.cuda.device_count() > 1:
print("{} GPUs are available. Let's use them.".format(
torch.cuda.device_count()))
model = torch.nn.DataParallel(model)
args.model = model
total_loss = 0.
result = 0.0
best_result = 0.0
print("Start training.")
for epoch in range(1, args.epochs_num + 1):
model.train()
for i, (src_batch, seg_batch, start_position_batch,
end_position_batch) in enumerate(
batch_loader(batch_size, src, seg, start_position,
end_position)):
loss = train(args, model, optimizer, scheduler, src_batch,
seg_batch, start_position_batch, end_position_batch)
total_loss += loss.item()
if (i + 1) % args.report_steps == 0:
print("Epoch id: {}, Training steps: {}, Avg loss: {:.3f}".
format(epoch, i + 1, total_loss / args.report_steps))
total_loss = 0.
result = evaluate(args, *read_dataset(args, args.dev_path))
if result > best_result:
best_result = result
save_model(model, args.output_model_path)
# Evaluation phase.
if args.test_path is not None:
print("Test set evaluation.")
if torch.cuda.device_count() > 1:
model.module.load_state_dict(torch.load(args.output_model_path))
else:
model.load_state_dict(torch.load(args.output_model_path))
evaluate(args, *read_dataset(args, args.test_path))
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# Path options.
parser.add_argument("--pretrained_model_path",
default=None,
type=str,
help="Path of the pretrained model.")
parser.add_argument("--dataset_path_list",
default=[],
nargs='+',
type=str,
help="Dataset path list.")
parser.add_argument("--output_model_path",
default="./models/multitask_classifier_model.bin",
type=str,
help="Path of the output model.")
parser.add_argument("--vocab_path",
default=None,
type=str,
help="Path of the vocabulary file.")
parser.add_argument("--spm_model_path",
default=None,
type=str,
help="Path of the sentence piece model.")
parser.add_argument("--config_path",
default="./models/bert_base_config.json",
type=str,
help="Path of the config file.")
# Model options.
parser.add_argument("--batch_size",
type=int,
default=32,
help="Batch size.")
parser.add_argument("--seq_length",
type=int,
default=128,
help="Sequence length.")
parser.add_argument("--embedding",
choices=["bert", "word"],
default="bert",
help="Emebdding type.")
parser.add_argument("--encoder", choices=["bert", "lstm", "gru", \
"cnn", "gatedcnn", "attn", "synt", \
"rcnn", "crnn", "gpt", "bilstm"], \
default="bert", help="Encoder type.")
parser.add_argument("--bidirectional",
action="store_true",
help="Specific to recurrent model.")
parser.add_argument("--pooling",
choices=["mean", "max", "first", "last"],
default="first",
help="Pooling type.")
parser.add_argument("--factorized_embedding_parameterization",
action="store_true",
help="Factorized embedding parameterization.")
parser.add_argument("--parameter_sharing",
action="store_true",
help="Parameter sharing.")
# Tokenizer options.
parser.add_argument(
"--tokenizer",
choices=["bert", "char", "space"],
default="bert",
help="Specify the tokenizer."
"Original Google BERT uses bert tokenizer on Chinese corpus."
"Char tokenizer segments sentences into characters."
"Space tokenizer segments sentences into words according to space.")
# Optimizer options.
parser.add_argument("--soft_targets",
action='store_true',
help="Train model with logits.")
parser.add_argument("--learning_rate",
type=float,
default=2e-5,
help="Learning rate.")
parser.add_argument("--warmup",
type=float,
default=0.1,
help="Warm up value.")
parser.add_argument(
"--fp16",
action='store_true',
help=
"Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit."
)
parser.add_argument(
"--fp16_opt_level",
choices=["O0", "O1", "O2", "O3"],
default='O1',
help=
"For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
"See details at https://nvidia.github.io/apex/amp.html")
# Training options.
parser.add_argument("--dropout", type=float, default=0.5, help="Dropout.")
parser.add_argument("--epochs_num",
type=int,
default=3,
help="Number of epochs.")
parser.add_argument("--report_steps",
type=int,
default=100,
help="Specific steps to print prompt.")
parser.add_argument("--seed", type=int, default=7, help="Random seed.")
args = parser.parse_args()
# Load the hyperparameters from the config file.
args = load_hyperparam(args)
set_seed(args.seed)
# Count the number of labels.
args.labels_num_list = [
count_labels_num(os.path.join(path, "train.tsv"))
for path in args.dataset_path_list
]
args.datasets_num = len(args.dataset_path_list)
# Build tokenizer.
args.tokenizer = globals()[args.tokenizer.capitalize() + "Tokenizer"](args)
# Build multi-task classification model.
model = MultitaskClassifier(args)
# Load or initialize parameters.
load_or_initialize_parameters(args, model)
args.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = model.to(args.device)
args.model = model
# Training phase.
dataset_list = [
read_dataset(args, os.path.join(path, "train.tsv"))
for path in args.dataset_path_list
]
packed_dataset_list = [
pack_dataset(dataset, i, args.batch_size)
for i, dataset in enumerate(dataset_list)
]
packed_dataset_all = []
for packed_dataset in packed_dataset_list:
packed_dataset_all += packed_dataset
random.shuffle(packed_dataset_all)
instances_num = sum([len(dataset) for dataset in dataset_list])
batch_size = args.batch_size
args.train_steps = int(instances_num * args.epochs_num / batch_size) + 1
print("Batch size: ", batch_size)
print("The number of training instances:", instances_num)
optimizer, scheduler = build_optimizer(args, model)
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
model, optimizer = amp.initialize(model,
optimizer,
opt_level=args.fp16_opt_level)
args.amp = amp
if torch.cuda.device_count() > 1:
print("{} GPUs are available. Let's use them.".format(
torch.cuda.device_count()))
model = torch.nn.DataParallel(model)
total_loss, result, best_result = 0., 0., 0.
print("Start training.")
for epoch in range(1, args.epochs_num + 1):
model.train()
for i, (dataset_id, src_batch, tgt_batch,
seg_batch) in enumerate(packed_dataset_all):
if hasattr(model, "module"):
model.module.change_dataset(dataset_id)
else:
model.change_dataset(dataset_id)
loss = train_model(args, model, optimizer, scheduler, src_batch,
tgt_batch, seg_batch, None)
total_loss += loss.item()
if (i + 1) % args.report_steps == 0:
print("Epoch id: {}, Training steps: {}, Avg loss: {:.3f}".
format(epoch, i + 1, total_loss / args.report_steps))
total_loss = 0.
for dataset_id, path in enumerate(args.dataset_path_list):
args.labels_num = args.labels_num_list[dataset_id]
if hasattr(model, "module"):
model.module.change_dataset(dataset_id)
else:
model.change_dataset(dataset_id)
result = evaluate(
args, read_dataset(args, os.path.join(path, "dev.tsv")))
save_model(model, args.output_model_path)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# Path options.
parser.add_argument("--pretrained_model_path",
default=None,
type=str,
help="Path of the pretrained model.")
parser.add_argument("--output_model_path",
default="./models/multichoice_model.bin",
type=str,
help="Path of the output model.")
parser.add_argument("--vocab_path",
default=None,
type=str,
help="Path of the vocabulary file.")
parser.add_argument("--spm_model_path",
default=None,
type=str,
help="Path of the sentence piece model.")
parser.add_argument("--train_data_path",
type=str,
required=True,
help="Path of the trainset.")
parser.add_argument("--train_label_path",
type=str,
required=True,
help="Path of the trainset.")
parser.add_argument("--dev_data_path",
type=str,
required=True,
help="Path of the devset.")
parser.add_argument("--dev_label_path",
type=str,
required=True,
help="Path of the devset.")
parser.add_argument("--config_path",
default="./models/bert_base_config.json",
type=str,
help="Path of the config file.")
# Model options.
parser.add_argument("--batch_size",
type=int,
default=32,
help="Batch size.")
parser.add_argument("--seq_length",
type=int,
default=512,
help="Sequence length.")
parser.add_argument("--embedding",
choices=["bert", "word"],
default="bert",
help="Emebdding type.")
parser.add_argument("--encoder", choices=["bert", "lstm", "gru", \
"cnn", "gatedcnn", "attn", "synt", \
"rcnn", "crnn", "gpt", "bilstm"], \
default="bert", help="Encoder type.")
parser.add_argument("--bidirectional",
action="store_true",
help="Specific to recurrent model.")
parser.add_argument("--factorized_embedding_parameterization",
action="store_true",
help="Factorized embedding parameterization.")
parser.add_argument("--parameter_sharing",
action="store_true",
help="Parameter sharing.")
parser.add_argument(
"--max_choices_num",
default=10,
type=int,
help=
"The maximum number of cadicate answer, shorter than this will be padded."
)
# Tokenizer options.
parser.add_argument(
"--tokenizer",
choices=["bert", "char", "space"],
default="bert",
help="Specify the tokenizer."
"Original Google BERT uses bert tokenizer on Chinese corpus."
"Char tokenizer segments sentences into characters."
"Space tokenizer segments sentences into words according to space.")
# Optimizer options.
parser.add_argument("--learning_rate",
type=float,
default=2e-5,
help="Learning rate.")
parser.add_argument("--warmup",
type=float,
default=0.1,
help="Warm up value.")
parser.add_argument(
"--fp16",
action='store_true',
help=
"Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit."
)
parser.add_argument(
"--fp16_opt_level",
choices=["O0", "O1", "O2", "O3"],
default='O1',
help=
"For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
"See details at https://nvidia.github.io/apex/amp.html")
# Training options.
parser.add_argument("--dropout", type=float, default=0.2, help="Dropout.")
parser.add_argument("--epochs_num",
type=int,
default=8,
help="Number of epochs.")
parser.add_argument("--report_steps",
type=int,
default=100,
help="Specific steps to print prompt.")
parser.add_argument("--seed", type=int, default=7, help="Random seed.")
args = parser.parse_args()
args.labels_num = args.max_choices_num
# Load the hyperparameters from the config file.
args = load_hyperparam(args)
set_seed(args.seed)
# Build tokenizer.
args.tokenizer = globals()[args.tokenizer.capitalize() + "Tokenizer"](args)
# Build multiple choice model.
model = MultipleChoice(args)
# Load or initialize parameters.
load_or_initialize_parameters(args, model)
args.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = model.to(args.device)
# Training phase.
trainset = read_dataset(args, args.train_data_path, args.train_label_path)
random.shuffle(trainset)
instances_num = len(trainset)
batch_size = args.batch_size
src = torch.LongTensor([example[0] for example in trainset])
tgt = torch.LongTensor([example[1] for example in trainset])
seg = torch.LongTensor([example[2] for example in trainset])
args.train_steps = int(instances_num * args.epochs_num / batch_size) + 1
print("Batch size: ", batch_size)
print("The number of training instances:", instances_num)
optimizer, scheduler = build_optimizer(args, model)
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
model, optimizer = amp.initialize(model,
optimizer,
opt_level=args.fp16_opt_level)
args.amp = amp
if torch.cuda.device_count() > 1:
print("{} GPUs are available. Let's use them.".format(
torch.cuda.device_count()))
model = torch.nn.DataParallel(model)
args.model = model
total_loss, result, best_result = 0., 0., 0.
print("Start training.")
for epoch in range(1, args.epochs_num + 1):
model.train()
for i, (src_batch, tgt_batch, seg_batch,
_) in enumerate(batch_loader(batch_size, src, tgt, seg)):
loss = train_model(args, model, optimizer, scheduler, src_batch,
tgt_batch, seg_batch)
total_loss += loss.item()
if (i + 1) % args.report_steps == 0:
print("Epoch id: {}, Training steps: {}, Avg loss: {:.3f}".
format(epoch, i + 1, total_loss / args.report_steps))
total_loss = 0.
result = evaluate(
args, read_dataset(args, args.dev_data_path, args.dev_label_path))
if result > best_result:
best_result = result
save_model(model, args.output_model_path)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
finetune_opts(parser)
parser.add_argument("--label2id_path",
type=str,
required=True,
help="Path of the label2id file.")
args = parser.parse_args()
# Load the hyperparameters of the config file.
args = load_hyperparam(args)
set_seed(args.seed)
args.begin_ids = []
with open(args.label2id_path, mode="r", encoding="utf-8") as f:
l2i = json.load(f)
print("Labels: ", l2i)
l2i["[PAD]"] = len(l2i)
for label in l2i:
if label.startswith("B"):
args.begin_ids.append(l2i[label])
args.l2i = l2i
args.labels_num = len(l2i)
args.tokenizer = SpaceTokenizer(args)
# Build sequence labeling model.
model = NerTagger(args)
# Load or initialize parameters.
load_or_initialize_parameters(args, model)
args.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = model.to(args.device)
# Training phase.
instances = read_dataset(args, args.train_path)
src = torch.LongTensor([ins[0] for ins in instances])
tgt = torch.LongTensor([ins[1] for ins in instances])
seg = torch.LongTensor([ins[2] for ins in instances])
instances_num = src.size(0)
batch_size = args.batch_size
args.train_steps = int(instances_num * args.epochs_num / batch_size) + 1
print("Batch size: ", batch_size)
print("The number of training instances:", instances_num)
optimizer, scheduler = build_optimizer(args, model)
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
model, optimizer = amp.initialize(model,
optimizer,
opt_level=args.fp16_opt_level)
if torch.cuda.device_count() > 1:
print("{} GPUs are available. Let's use them.".format(
torch.cuda.device_count()))
model = torch.nn.DataParallel(model)
args.model = model
total_loss, f1, best_f1 = 0.0, 0.0, 0.0
print("Start training.")
for epoch in range(1, args.epochs_num + 1):
model.train()
for i, (src_batch, tgt_batch,
seg_batch) in enumerate(batch_loader(batch_size, src, tgt,
seg)):
loss = train(args, model, optimizer, scheduler, src_batch,
tgt_batch, seg_batch)
total_loss += loss.item()
if (i + 1) % args.report_steps == 0:
print("Epoch id: {}, Training steps: {}, Avg loss: {:.3f}".
format(epoch, i + 1, total_loss / args.report_steps))
total_loss = 0.
f1 = evaluate(args, read_dataset(args, args.dev_path))
if f1 > best_f1:
best_f1 = f1
save_model(model, args.output_model_path)
else:
continue
# Evaluation phase.
if args.test_path is not None:
print("Test set evaluation.")
if torch.cuda.device_count() > 1:
model.module.load_state_dict(torch.load(args.output_model_path))
else:
model.load_state_dict(torch.load(args.output_model_path))
evaluate(args, read_dataset(args, args.test_path))
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# Path options.
parser.add_argument("--pretrained_model_path",
default=None,
type=str,
help="Path of the pretrained model.")
parser.add_argument("--output_model_path",
default="./models/ner_model.bin",
type=str,
help="Path of the output model.")
parser.add_argument("--vocab_path",
default=None,
type=str,
help="Path of the vocabulary file.")
parser.add_argument("--spm_model_path",
default=None,
type=str,
help="Path of the sentence piece model.")
parser.add_argument("--train_path",
type=str,
required=True,
help="Path of the trainset.")
parser.add_argument("--dev_path",
type=str,
required=True,
help="Path of the devset.")
parser.add_argument("--test_path", type=str, help="Path of the testset.")
parser.add_argument("--config_path",
default="./models/bert_base_config.json",
type=str,
help="Path of the config file.")
parser.add_argument("--label2id_path",
type=str,
required=True,
help="Path of the label2id file.")
# Model options.
parser.add_argument("--batch_size",
type=int,
default=32,
help="Batch_size.")
parser.add_argument("--seq_length",
default=128,
type=int,
help="Sequence length.")
parser.add_argument("--embedding",
choices=["bert", "word"],
default="bert",
help="Emebdding type.")
parser.add_argument("--encoder", choices=["bert", "lstm", "gru", \
"cnn", "gatedcnn", "attn", "synt", \
"rcnn", "crnn", "gpt", "bilstm"], \
default="bert", help="Encoder type.")
parser.add_argument("--bidirectional",
action="store_true",
help="Specific to recurrent model.")
parser.add_argument("--factorized_embedding_parameterization",
action="store_true",
help="Factorized embedding parameterization.")
parser.add_argument("--parameter_sharing",
action="store_true",
help="Parameter sharing.")
# Optimizer options.
parser.add_argument("--learning_rate",
type=float,
default=2e-5,
help="Learning rate.")
parser.add_argument("--warmup",
type=float,
default=0.1,
help="Warm up value.")
parser.add_argument(
"--fp16",
action='store_true',
help=
"Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"
)
parser.add_argument(
"--fp16_opt_level",
choices=["O0", "O1", "O2", "O3"],
default='O1',
help=
"For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
"See details at https://nvidia.github.io/apex/amp.html")
# Training options.
parser.add_argument("--dropout", type=float, default=0.1, help="Dropout.")
parser.add_argument("--epochs_num",
type=int,
default=3,
help="Number of epochs.")
parser.add_argument("--report_steps",
type=int,
default=100,
help="Specific steps to print prompt.")
parser.add_argument("--seed", type=int, default=7, help="Random seed.")
args = parser.parse_args()
# Load the hyperparameters of the config file.
args = load_hyperparam(args)
set_seed(args.seed)
args.begin_ids = []
with open(args.label2id_path, mode="r", encoding="utf-8") as f:
l2i = json.load(f)
print("Labels: ", l2i)
l2i["[PAD]"] = len(l2i)
for label in l2i:
if label.startswith("B"):
args.begin_ids.append(l2i[label])
args.l2i = l2i
args.labels_num = len(l2i)
args.tokenizer = SpaceTokenizer(args)
# Build sequence labeling model.
model = NerTagger(args)
# Load or initialize parameters.
load_or_initialize_parameters(args, model)
args.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = model.to(args.device)
# Training phase.
instances = read_dataset(args, args.train_path)
src = torch.LongTensor([ins[0] for ins in instances])
tgt = torch.LongTensor([ins[1] for ins in instances])
seg = torch.LongTensor([ins[2] for ins in instances])
instances_num = src.size(0)
batch_size = args.batch_size
args.train_steps = int(instances_num * args.epochs_num / batch_size) + 1
print("Batch size: ", batch_size)
print("The number of training instances:", instances_num)
optimizer, scheduler = build_optimizer(args, model)
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
model, optimizer = amp.initialize(model,
optimizer,
opt_level=args.fp16_opt_level)
if torch.cuda.device_count() > 1:
print("{} GPUs are available. Let's use them.".format(
torch.cuda.device_count()))
model = torch.nn.DataParallel(model)
args.model = model
total_loss, f1, best_f1 = 0., 0., 0.
print("Start training.")
for epoch in range(1, args.epochs_num + 1):
model.train()
for i, (src_batch, tgt_batch,
seg_batch) in enumerate(batch_loader(batch_size, src, tgt,
seg)):
loss = train(args, model, optimizer, scheduler, src_batch,
tgt_batch, seg_batch)
total_loss += loss.item()
if (i + 1) % args.report_steps == 0:
print("Epoch id: {}, Training steps: {}, Avg loss: {:.3f}".
format(epoch, i + 1, total_loss / args.report_steps))
total_loss = 0.
f1 = evaluate(args, read_dataset(args, args.dev_path))
if f1 > best_f1:
best_f1 = f1
save_model(model, args.output_model_path)
else:
continue
# Evaluation phase.
if args.test_path is not None:
print("Test set evaluation.")
if torch.cuda.device_count() > 1:
model.module.load_state_dict(torch.load(args.output_model_path))
else:
model.load_state_dict(torch.load(args.output_model_path))
evaluate(args, read_dataset(args, args.test_path))
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
finetune_opts(parser)
parser.add_argument("--pooling",
choices=["mean", "max", "first", "last"],
default="first",
help="Pooling type.")
parser.add_argument(
"--tokenizer",
choices=["bert", "char", "space"],
default="bert",
help="Specify the tokenizer."
"Original Google BERT uses bert tokenizer on Chinese corpus."
"Char tokenizer segments sentences into characters."
"Space tokenizer segments sentences into words according to space.")
parser.add_argument("--soft_targets",
action='store_true',
help="Train model with logits.")
parser.add_argument("--soft_alpha",
type=float,
default=0.5,
help="Weight of the soft targets loss.")
args = parser.parse_args()
if args.output_model_path == None:
args.output_model_path = "./models/dbqa_model.bin"
# Load the hyperparameters from the config file.
args = load_hyperparam(args)
set_seed(args.seed)
# Count the number of labels.
args.labels_num = count_labels_num(args.train_path)
# Build tokenizer.
args.tokenizer = globals()[args.tokenizer.capitalize() + "Tokenizer"](args)
# Build classification model.
model = Classifier(args)
# Load or initialize parameters.
load_or_initialize_parameters(args, model)
args.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = model.to(args.device)
# Training phase.
trainset = read_dataset(args, args.train_path)
random.shuffle(trainset)
instances_num = len(trainset)
batch_size = args.batch_size
src = torch.LongTensor([example[0] for example in trainset])
tgt = torch.LongTensor([example[1] for example in trainset])
seg = torch.LongTensor([example[2] for example in trainset])
args.train_steps = int(instances_num * args.epochs_num / batch_size) + 1
print("Batch size: ", batch_size)
print("The number of training instances:", instances_num)
optimizer, scheduler = build_optimizer(args, model)
if args.fp16:
try:
from apex import amp
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
model, optimizer = amp.initialize(model,
optimizer,
opt_level=args.fp16_opt_level)
args.amp = amp
if torch.cuda.device_count() > 1:
print("{} GPUs are available. Let's use them.".format(
torch.cuda.device_count()))
model = torch.nn.DataParallel(model)
args.model = model
total_loss, result, best_result = 0., 0., 0.
print("Start training.")
for epoch in range(1, args.epochs_num + 1):
model.train()
for i, (src_batch, tgt_batch, seg_batch,
_) in enumerate(batch_loader(batch_size, src, tgt, seg)):
loss = train_model(args, model, optimizer, scheduler, src_batch,
tgt_batch, seg_batch)
total_loss += loss.item()
if (i + 1) % args.report_steps == 0:
print("Epoch id: {}, Training steps: {}, Avg loss: {:.3f}".
format(epoch, i + 1, total_loss / args.report_steps))
total_loss = 0.
result = evaluate(args, read_dataset(args, args.dev_path))
if result > best_result:
best_result = result
save_model(model, args.output_model_path)
# Evaluation phase.
if args.test_path is not None:
print("Test set evaluation.")
if torch.cuda.device_count() > 1:
model.module.load_state_dict(torch.load(args.output_model_path))
else:
model.load_state_dict(torch.load(args.output_model_path))
evaluate(args, read_dataset(args, args.test_path))
