def run_translation():
"""
run translation task
"""
parser = argparse.ArgumentParser(
description="Finetune and Evaluate translation")
parser.add_argument("--device_target",
type=str,
default="Ascend",
help="Device type. Default: Ascend.")
parser.add_argument("--device_id",
type=int,
default=0,
help="ID of target device. ")
parser.add_argument(
"--metric_method",
type=str,
default="BLEU",
help="The eval method including [BLEU]. Default: BLEU.")
parser.add_argument("--do_train",
type=str,
default="false",
help="Enable train. Default: false.")
parser.add_argument("--do_eval",
type=str,
default="true",
help="Enable evaluation. Default: false.")
parser.add_argument(
"--eval_type",
type=str,
default="zero-shot",
help=
"The type of evaluation including [zero-shot, finetuned]. Default: zero-shot."
)
parser.add_argument("--epoch_num",
type=int,
default=1,
help="Epoch number. Default: 1.")
parser.add_argument("--train_data_shuffle",
type=str,
default="true",
help="Enable train data shuffle. Default: true.")
parser.add_argument("--eval_data_shuffle",
type=str,
default="false",
help="Enable eval data shuffle. Default: false.")
parser.add_argument("--save_finetune_ckpt_path",
type=str,
default="",
help="Save the checkpoint path.")
parser.add_argument("--load_pretrain_ckpt_path",
type=str,
default="",
help="Load the checkpoint file path.")
parser.add_argument("--load_finetune_ckpt_path",
type=str,
default="",
help="Load the checkpoint file path.")
parser.add_argument("--train_data_file_path",
type=str,
default="",
help="Data path, it is better to use absolute path")
parser.add_argument("--eval_data_file_path",
type=str,
default="",
help="Data path, it is better to use absolute path")
parser.add_argument("--tokenizer_file_path",
type=str,
default="",
help="pretrained vocab and merge file path.")
parser.add_argument("--generate_length",
type=int,
default=150,
help="The generation length of translation sentence.")
parser.add_argument("--top_k",
type=int,
default=1,
help="Parameter for Top-K sampling.")
parser.add_argument("--top_p",
type=str,
default="1.0",
help="parameter for Top-P sampling.")
parser.add_argument(
"--temperature",
type=str,
default="1.0",
help="Parameter for generation, greater if generation more diverse. ")
args_opt = parser.parse_args()
epoch_num = args_opt.epoch_num
metric = args_opt.metric_method
save_finetune_ckpt_path = args_opt.save_finetune_ckpt_path
load_finetune_ckpt_path = args_opt.load_finetune_ckpt_path
load_pretrain_ckpt_path = args_opt.load_pretrain_ckpt_path
if args_opt.do_train.lower() == "false" and args_opt.do_eval.lower(
) == "false":
raise ValueError(
"At least one of 'do_train' or 'do_eval' must be true")
if args_opt.do_train.lower(
) == "true" and args_opt.train_data_file_path == "":
raise ValueError(
"'train_data_file_path' must be set when do finetune task")
if args_opt.do_eval.lower(
) == "true" and args_opt.eval_data_file_path == "":
raise ValueError(
"'eval_data_file_path' must be set when do evaluation task")
device_target = args_opt.device_target
if device_target == "Ascend":
context.set_context(mode=context.GRAPH_MODE,
device_target=device_target,
device_id=args_opt.device_id,
max_call_depth=3000)
context.set_auto_parallel_context(parallel_mode="stand_alone")
print(" | Device: {} | Device id: {}".format(device_target,
args_opt.device_id))
else:
raise Exception("Device target error, Ascend is supported.")
gpt2_loss = GPT2Translation(config=gpt2_net_cfg,
is_training=True,
use_one_hot_embeddings=False)
if args_opt.do_train.lower() == "true":
get_train_setting(cfg)
get_model_setting(cfg, gpt2_net_cfg)
print(
"============== Start Loading Translation Train Dataset =============="
)
print(" | Train Dataset: {}".format(args_opt.train_data_file_path))
print(" | Checkpoint: {}".format(args_opt.load_pretrain_ckpt_path))
train_dataset = create_language_model_dataset(
do_shuffle=(args_opt.train_data_shuffle.lower() == "true"),
dataset_path=args_opt.train_data_file_path)
do_train(train_dataset, gpt2_loss, load_pretrain_ckpt_path,
save_finetune_ckpt_path, epoch_num)
if args_opt.do_eval.lower() == "true":
get_model_setting(cfg, gpt2_net_cfg)
print(
"============ Start Loading Translation Evaluation Dataset ============"
)
print(" | Eval Dataset: {}".format(args_opt.eval_data_file_path))
print(" | Checkpoint: {}".format(args_opt.load_finetune_ckpt_path))
eval_dataset = create_language_model_dataset(
do_shuffle=(args_opt.eval_data_shuffle.lower() == "true"),
dataset_path=args_opt.eval_data_file_path)
do_eval(eval_dataset, GPT2TranslationModel, metric,
load_finetune_ckpt_path, args_opt.eval_type,
args_opt.tokenizer_file_path, args_opt.generate_length,
args_opt.top_k, args_opt.top_p, args_opt.temperature)
def run_summarization():
"""
Run Summarization task.
"""
# set argument parser
parser = argparse.ArgumentParser(
description="Finetune and Evaluate Summrization")
# context and task settings
parser.add_argument("--device_target",
type=str,
default="Ascend",
help="Device type. Default: Ascend.")
parser.add_argument("--device_id",
type=int,
default=4,
help="ID of target device.")
parser.add_argument("--do_train",
type=str,
default="false",
help="Enable train. Default: false.")
parser.add_argument("--do_eval",
type=str,
default="true",
help="Enable evaluation. Default: false.")
parser.add_argument(
"--eval_type",
type=str,
default="finetuned",
help=
"The type of evaluation including [zero-shot, finetuned]. Default: zero-shot."
)
parser.add_argument(
"--metric_method",
type=str,
default="Rouge",
help=
"The eval method including [Rouge(Rouge1,Rouge2,RougeL,Rouge Avg)]. Default: Rouge."
)
parser.add_argument("--epoch_num",
type=int,
default=2,
help="Epoch number. Default: 2.")
# dataset and params_dict file settings
parser.add_argument("--train_data_shuffle",
type=str,
default="true",
help="Enable train data shuffle. Default: true.")
parser.add_argument("--eval_data_shuffle",
type=str,
default="false",
help="Enable eval data shuffle. Default: false.")
parser.add_argument("--save_finetune_ckpt_path",
type=str,
default="",
help="Save the checkpoint path.")
parser.add_argument("--load_pretrain_ckpt_path",
type=str,
default="",
help="Load the checkpoint file path.")
parser.add_argument("--load_finetune_ckpt_path",
type=str,
default="",
help="Load the checkpoint file path.")
parser.add_argument("--train_data_file_path",
type=str,
default="",
help="Data path, it is better to use absolute path")
parser.add_argument("--eval_data_file_path",
type=str,
default="",
help="Data path, it is better to use absolute path")
# sampling settings
parser.add_argument("--top_k",
type=int,
default=2,
help="top k tokens chosen for sampling")
parser.add_argument(
"--top_p",
type=str,
default="1.0",
help="top p accumulated probability threshold for logit to be counted")
parser.add_argument("--generate_length",
type=int,
default=100,
help="the number of generated tokens.")
parser.add_argument("--temperature",
type=str,
default="1.0",
help="temperature on logits for sampling")
parser.add_argument("--tokenizer_file_path",
type=str,
default="",
help="vocab & merge file path")
args_opt = parser.parse_args()
epoch_num = args_opt.epoch_num
metric = args_opt.metric_method
save_finetune_ckpt_path = args_opt.save_finetune_ckpt_path
load_finetune_ckpt_path = args_opt.load_finetune_ckpt_path
load_pretrain_ckpt_path = args_opt.load_pretrain_ckpt_path
eval_type = args_opt.eval_type
tokenizer_file = args_opt.tokenizer_file_path
if args_opt.do_train.lower() == "false" and args_opt.do_eval.lower(
) == "false":
raise ValueError(
"At least one of 'do_train' or 'do_eval' must be true")
if args_opt.do_train.lower(
) == "true" and args_opt.train_data_file_path == "":
raise ValueError(
"'train_data_file_path' must be set when do finetune task")
if args_opt.do_eval.lower(
) == "true" and args_opt.eval_data_file_path == "":
raise ValueError(
"'eval_data_file_path' must be set when do evaluation task")
device = args_opt.device_target
if device == "Ascend":
context.set_context(mode=context.GRAPH_MODE,
device_target="Ascend",
device_id=args_opt.device_id)
context.set_auto_parallel_context(parallel_mode="stand_alone")
print(" | Device: {} | Device id: {}".format(device,
args_opt.device_id))
else:
raise Exception("Device target error, Ascend is supported.")
if args_opt.do_train.lower() == "true":
get_train_setting(cfg)
get_model_setting(cfg, gpt2_net_cfg)
train_data_file_path = args_opt.train_data_file_path
gpt2_loss = GPT2Summarization(config=gpt2_net_cfg,
is_training=True,
use_one_hot_embeddings=False)
print("============== Start Loading Train Dataset ============")
train_dataset = create_language_model_dataset(
do_shuffle=(args_opt.train_data_shuffle.lower() == "true"),
dataset_path=train_data_file_path)
do_train(train_dataset, gpt2_loss, load_pretrain_ckpt_path,
save_finetune_ckpt_path, epoch_num)
if args_opt.do_eval.lower() == "true":
get_model_setting(cfg, gpt2_net_cfg)
eval_dataset_file_path = args_opt.eval_data_file_path
print("============== Start Loading Evaluation Dataset ============")
eval_dataset = create_language_model_dataset(
do_shuffle=(args_opt.train_data_shuffle.lower() == "true"),
dataset_path=eval_dataset_file_path)
do_eval(eval_dataset, GPT2SummarizationModel, metric,
load_finetune_ckpt_path, eval_type, tokenizer_file,
args_opt.top_k, args_opt.top_p, args_opt.temperature,
args_opt.generate_length)
def run_cbt_task():
"""
run Children's Book Test (CBT) task
"""
parser = argparse.ArgumentParser(
description="Finetune and Evaluate CBT task")
parser.add_argument("--device_target",
type=str,
default="Ascend",
help="Device type. Default: Ascend.")
parser.add_argument("--device_id",
type=int,
default=1,
help="ID of target device. ")
parser.add_argument("--num_choice",
type=int,
default=10,
help="The number of choice in CBT task. ")
parser.add_argument(
"--metric_method",
type=str,
default="Accuracy",
help="The eval method including [Accuracy]. Default: Accuracy.")
parser.add_argument("--do_train",
type=str,
default="false",
help="Enable train. Default: false.")
parser.add_argument("--do_eval",
type=str,
default="true",
help="Enable evaluation. Default: true.")
parser.add_argument(
"--eval_type",
type=str,
default="zero-shot",
help=
"The type of evaluation including [zero-shot, finetuned]. Default: zero-shot."
)
parser.add_argument("--epoch_num",
type=int,
default=1,
help="Epoch number. Default: 1.")
parser.add_argument("--train_data_shuffle",
type=str,
default="true",
help="Enable train data shuffle. Default: true.")
parser.add_argument("--eval_data_shuffle",
type=str,
default="false",
help="Enable eval data shuffle. Default: false.")
parser.add_argument("--save_finetune_ckpt_path",
type=str,
default="",
help="Save the finetuned checkpoint path.")
parser.add_argument("--load_pretrain_ckpt_path",
type=str,
default="",
help="Load the checkpoint file path for train.")
parser.add_argument("--load_finetune_ckpt_path",
type=str,
default="",
help="Load the checkpoint file path for evaluation.")
parser.add_argument("--train_data_file_path",
type=str,
default="",
help="Data path, it is better to use absolute path")
parser.add_argument("--eval_data_file_path",
type=str,
default="",
help="Data path, it is better to use absolute path")
args_opt = parser.parse_args()
epoch_num = args_opt.epoch_num
metric = args_opt.metric_method
save_finetune_ckpt_path = args_opt.save_finetune_ckpt_path
load_finetune_ckpt_path = args_opt.load_finetune_ckpt_path
load_pretrain_ckpt_path = args_opt.load_pretrain_ckpt_path
if args_opt.do_train.lower() == "false" and args_opt.do_eval.lower(
) == "false":
raise ValueError(
"At least one of 'do_train' or 'do_eval' must be true")
if args_opt.do_train.lower(
) == "true" and args_opt.train_data_file_path == "":
raise ValueError(
"'train_data_file_path' must be set when do finetune task")
if args_opt.do_eval.lower(
) == "true" and args_opt.eval_data_file_path == "":
raise ValueError(
"'eval_data_file_path' must be set when do evaluation task")
device_target = args_opt.device_target
if device_target == "Ascend":
context.set_context(mode=context.GRAPH_MODE,
device_target=device_target,
device_id=args_opt.device_id,
max_call_depth=3000)
context.set_auto_parallel_context(parallel_mode="stand_alone")
print(" | Device: {} | Device id: {}".format(device_target,
args_opt.device_id))
else:
raise Exception("Device target error, Ascend is supported.")
gpt2_loss = GPT2CBT(config=gpt2_net_cfg,
is_training=True,
use_one_hot_embeddings=False)
if args_opt.do_train.lower() == "true":
get_train_setting(cfg)
get_model_setting(cfg, gpt2_net_cfg)
print("============== Start Loading Train Dataset ============")
print(" | Train Dataset: {}".format(args_opt.train_data_file_path))
print(" | Checkpoint: {}".format(args_opt.load_pretrain_ckpt_path))
train_dataset = create_language_model_dataset(
do_shuffle=(args_opt.train_data_shuffle.lower() == "true"),
dataset_path=args_opt.train_data_file_path)
do_train(train_dataset, gpt2_loss, load_pretrain_ckpt_path,
save_finetune_ckpt_path, epoch_num)
if args_opt.do_eval.lower() == "true":
get_model_setting(cfg, gpt2_net_cfg)
print("============== Start Loading Evaluation Dataset ============")
print(" | Eval Dataset: {}".format(args_opt.eval_data_file_path))
print(" | Checkpoint: {}".format(args_opt.load_finetune_ckpt_path))
eval_dataset = create_cbt_dataset(
do_shuffle=(args_opt.eval_data_shuffle.lower() == "true"),
dataset_path=args_opt.eval_data_file_path)
do_eval(eval_dataset, GPT2CBT, metric, load_finetune_ckpt_path,
args_opt.eval_type, args_opt.num_choice)
def run_lambada():
"""
Run Lambada task.
"""
parser = argparse.ArgumentParser(
description="Finetune and Evaluate languagemodel")
parser.add_argument("--device_target",
type=str,
default="Ascend",
help="Device type. Default: Ascend.")
parser.add_argument("--device_id",
type=int,
default=2,
help="ID of target device.")
parser.add_argument(
"--metric_method",
type=str,
default="PPL",
help="The eval method including [Accuracy, PPL]. Default: Accuracy.")
parser.add_argument("--do_train",
type=str,
default="false",
help="Enable train. Default: false.")
parser.add_argument("--do_eval",
type=str,
default="true",
help="Enable evaluation. Default: false.")
parser.add_argument(
"--eval_type",
type=str,
default="finetuned",
help=
"The type of evaluation including [zero-shot, finetuned]. Default: zero-shot."
)
parser.add_argument("--epoch_num",
type=int,
default=3,
help="Epoch number. Default: 1.")
parser.add_argument("--train_data_shuffle",
type=str,
default="false",
help="Enable train data shuffle. Default: true.")
parser.add_argument("--eval_data_shuffle",
type=str,
default="false",
help="Enable eval data shuffle. Default: false.")
parser.add_argument(
"--generate_length_dynamically",
type=str,
default="true",
help="Enable generate_length_Dynamically. Default: true.")
parser.add_argument("--save_finetune_ckpt_path",
type=str,
default="",
help="Save the checkpoint path.")
parser.add_argument("--load_pretrain_ckpt_path",
type=str,
default="",
help="Load the checkpoint file path.")
parser.add_argument("--load_finetune_ckpt_path",
type=str,
default="",
help="Load the checkpoint file path.")
parser.add_argument("--train_data_file_path",
type=str,
default="",
help="Data path, it is better to use absolute path.")
parser.add_argument("--eval_data_file_path",
type=str,
default="",
help="Data path, it is better to use absolute path.")
parser.add_argument("--tokenizer_file_path",
type=str,
default="",
help="pretrained vocab and merge file path.")
parser.add_argument("--stop_word_file_path",
type=str,
default="",
help="The stop word file path.")
args_opt = parser.parse_args()
epoch_num = args_opt.epoch_num
metric = args_opt.metric_method
save_finetune_ckpt_path = args_opt.save_finetune_ckpt_path
load_finetune_ckpt_path = args_opt.load_finetune_ckpt_path
load_pretrain_ckpt_path = args_opt.load_pretrain_ckpt_path
if args_opt.do_train.lower() == "false" and args_opt.do_eval.lower(
) == "false":
raise ValueError(
"At least one of 'do_train' or 'do_eval' must be true")
if args_opt.do_train.lower(
) == "true" and args_opt.train_data_file_path == "":
raise ValueError(
"'train_data_file_path' must be set when do finetune task")
if args_opt.do_eval.lower(
) == "true" and args_opt.eval_data_file_path == "":
raise ValueError(
"'eval_data_file_path' must be set when do evaluation task")
device = args_opt.device_target
if device == "Ascend":
context.set_context(mode=context.GRAPH_MODE,
device_target="Ascend",
device_id=args_opt.device_id)
context.set_auto_parallel_context(parallel_mode="stand_alone")
print(" | Device: {} | Device id: {}".format(device,
args_opt.device_id))
else:
raise Exception("Device target error, Ascend is supported.")
gpt2_loss = GPT2Lambada(config=gpt2_net_cfg,
is_training=True,
use_one_hot_embeddings=False)
if args_opt.do_train.lower() == "true":
get_train_setting(cfg)
get_model_setting(cfg, gpt2_net_cfg)
print("============== Start Loading Train Dataset ============")
print(" | Train Dataset: {}".format(args_opt.train_data_file_path))
print(" | Checkpoint: {}".format(args_opt.load_pretrain_ckpt_path))
train_dataset = create_language_model_dataset(
do_shuffle=(args_opt.train_data_shuffle.lower() == "true"),
dataset_path=args_opt.train_data_file_path)
do_train(train_dataset, gpt2_loss, load_pretrain_ckpt_path,
save_finetune_ckpt_path, epoch_num)
if args_opt.do_eval.lower() == "true":
get_model_setting(cfg, gpt2_net_cfg)
print("============== Start Loading Evaluation Dataset ============")
print(" | Eval Dataset: {}".format(args_opt.eval_data_file_path))
print(" | Checkpoint: {}".format(args_opt.load_finetune_ckpt_path))
eval_dataset = create_lambada_control_dataset(
do_shuffle=(args_opt.eval_data_shuffle.lower() == "true"),
dataset_path=args_opt.eval_data_file_path)
do_eval(eval_dataset, GPT2Lambada, metric, load_finetune_ckpt_path,
args_opt.eval_type, args_opt.stop_word_file_path,
args_opt.generate_length_dynamically,
args_opt.tokenizer_file_path)