import paddle.fluid as fluid from collections import namedtuple sys.path.append("..") print(sys.path) from preprocess.ernie import task_reader from models.representation.ernie import ErnieConfig from models.representation.ernie import ernie_encoder #from models.representation.ernie import ernie_pyreader from models.sequence_labeling import nets import utils # yapf: disable parser = argparse.ArgumentParser(__doc__) model_g = utils.ArgumentGroup(parser, "model", "model configuration and paths.") model_g.add_arg("ernie_config_path", str, "../LARK/ERNIE/config/ernie_config.json", "Path to the json file for ernie model config.") model_g.add_arg("lac_config_path", str, None, "Path to the json file for LAC model config.") model_g.add_arg("init_checkpoint", str, None, "Init checkpoint to resume training from.") model_g.add_arg("checkpoints", str, None, "Path to save checkpoints") model_g.add_arg("init_pretraining_params", str, "pretrained/params/", "Init pre-training params which preforms fine-tuning from. If the " "arg 'init_checkpoint' has been set, this argument wouldn't be valid.") train_g = utils.ArgumentGroup(parser, "training", "training options.") train_g.add_arg("epoch", int, 10, "Number of epoches for training.") train_g.add_arg("save_steps", int, 10000, "The steps interval to save checkpoints.") train_g.add_arg("validation_steps", int, 1000, "The steps interval to evaluate model performance.") train_g.add_arg("lr", float, 0.001, "The Learning rate value for training.") train_g.add_arg("crf_learning_rate", float, 0.2,
import numpy as np import paddle import paddle.fluid as fluid import reader import utils sys.path.append("../") from models.sequence_labeling import nets # yapf: disable parser = argparse.ArgumentParser(__doc__) # 1. model parameters model_g = utils.ArgumentGroup(parser, "model", "model configuration") model_g.add_arg("word_emb_dim", int, 128, "The dimension in which a word is embedded.") model_g.add_arg("grnn_hidden_dim", int, 256, "The number of hidden nodes in the GRNN layer.") model_g.add_arg("bigru_num", int, 2, "The number of bi_gru layers in the network.") # 2. data parameters data_g = utils.ArgumentGroup(parser, "data", "data paths") data_g.add_arg("word_dict_path", str, "./conf/word.dic", "The path of the word dictionary.") data_g.add_arg("label_dict_path", str, "./conf/tag.dic", "The path of the label dictionary.") data_g.add_arg("word_rep_dict_path", str, "./conf/q2b.dic", "The path of the word replacement Dictionary.") data_g.add_arg("train_data", str, "./data/train.tsv", "The folder where the training data is located.") data_g.add_arg("test_data", str, "./data/test.tsv", "The folder where the training data is located.") data_g.add_arg("infer_data", str, "./data/test.tsv", "The folder where the training data is located.") data_g.add_arg("model_save_dir", str, "./models", "The model will be saved in this path.") data_g.add_arg("init_checkpoint", str, "", "Path to init model")
import argparse import numpy as np import paddle.fluid as fluid import utils import creator import reader parser = argparse.ArgumentParser(__doc__) # 1. model parameters model_g = utils.ArgumentGroup(parser, "model", "model configuration") model_g.add_arg("word_emb_dim", int, 128, "The dimension in which a word is embedded.") model_g.add_arg("grnn_hidden_dim", int, 128, "The number of hidden nodes in the GRNN layer.") model_g.add_arg("bigru_num", int, 2, "The number of bi_gru layers in the network.") model_g.add_arg("use_cuda", bool, False, "If set, use GPU for training.") # 2. data parameters data_g = utils.ArgumentGroup(parser, "data", "data paths") data_g.add_arg("word_dict_path", str, "./conf/word.dic", "The path of the word dictionary.") data_g.add_arg("label_dict_path", str, "./conf/tag.dic", "The path of the label dictionary.") data_g.add_arg("word_rep_dict_path", str, "./conf/q2b.dic", "The path of the word replacement Dictionary.") data_g.add_arg("test_data", str, "./data/test.tsv", "The folder where the training data is located.") data_g.add_arg("save_bin_path", str, "./data/test_eval_1000.bin",
train_feed_list=train_feed_list, train_fetch_list=train_fetch_list, eval_program=test_program, eval_reader=test_reader, eval_feed_list=test_feed_list, eval_fetch_list=test_fetch_list, teacher_programs=[], train_optimizer=optimizer, distiller_optimizer=None) com_pass.config(args.compress_config) com_pass.run() if __name__ == "__main__": # 参数控制可以根据需求使用argparse,yaml或者json # 对NLP任务推荐使用PALM下定义的configure,可以统一argparse,yaml或者json格式的配置文件。 parser = argparse.ArgumentParser(__doc__) utils.load_yaml(parser, 'conf/args.yaml') user_parser = utils.ArgumentGroup(parser, "model", "model configuration") user_parser.add_arg("compress_config", str, 'conf/quantization.yaml', "The compress configure file") args = parser.parse_args() check_cuda(args.use_cuda) print(args) do_compress(args)