import paddle.fluid as fluid
from collections import namedtuple
sys.path.append("..")
print(sys.path)
from preprocess.ernie import task_reader
from models.representation.ernie import ErnieConfig
from models.representation.ernie import ernie_encoder
#from models.representation.ernie import ernie_pyreader
from models.sequence_labeling import nets
import utils
# yapf: disable
parser = argparse.ArgumentParser(__doc__)
model_g = utils.ArgumentGroup(parser, "model", "model configuration and paths.")
model_g.add_arg("ernie_config_path", str, "../LARK/ERNIE/config/ernie_config.json",
"Path to the json file for ernie model config.")
model_g.add_arg("lac_config_path", str, None, "Path to the json file for LAC model config.")
model_g.add_arg("init_checkpoint", str, None, "Init checkpoint to resume training from.")
model_g.add_arg("checkpoints", str, None, "Path to save checkpoints")
model_g.add_arg("init_pretraining_params", str, "pretrained/params/",
"Init pre-training params which preforms fine-tuning from. If the "
"arg 'init_checkpoint' has been set, this argument wouldn't be valid.")
train_g = utils.ArgumentGroup(parser, "training", "training options.")
train_g.add_arg("epoch", int, 10, "Number of epoches for training.")
train_g.add_arg("save_steps", int, 10000, "The steps interval to save checkpoints.")
train_g.add_arg("validation_steps", int, 1000, "The steps interval to evaluate model performance.")
train_g.add_arg("lr", float, 0.001, "The Learning rate value for training.")
train_g.add_arg("crf_learning_rate", float, 0.2,
import numpy as np
import paddle
import paddle.fluid as fluid
import reader
import utils
sys.path.append("../")
from models.sequence_labeling import nets
# yapf: disable
parser = argparse.ArgumentParser(__doc__)
# 1. model parameters
model_g = utils.ArgumentGroup(parser, "model", "model configuration")
model_g.add_arg("word_emb_dim", int, 128, "The dimension in which a word is embedded.")
model_g.add_arg("grnn_hidden_dim", int, 256, "The number of hidden nodes in the GRNN layer.")
model_g.add_arg("bigru_num", int, 2, "The number of bi_gru layers in the network.")
# 2. data parameters
data_g = utils.ArgumentGroup(parser, "data", "data paths")
data_g.add_arg("word_dict_path", str, "./conf/word.dic", "The path of the word dictionary.")
data_g.add_arg("label_dict_path", str, "./conf/tag.dic", "The path of the label dictionary.")
data_g.add_arg("word_rep_dict_path", str, "./conf/q2b.dic", "The path of the word replacement Dictionary.")
data_g.add_arg("train_data", str, "./data/train.tsv", "The folder where the training data is located.")
data_g.add_arg("test_data", str, "./data/test.tsv", "The folder where the training data is located.")
data_g.add_arg("infer_data", str, "./data/test.tsv", "The folder where the training data is located.")
data_g.add_arg("model_save_dir", str, "./models", "The model will be saved in this path.")
data_g.add_arg("init_checkpoint", str, "", "Path to init model")
import argparse
import numpy as np
import paddle.fluid as fluid
import utils
import creator
import reader
parser = argparse.ArgumentParser(__doc__)
# 1. model parameters
model_g = utils.ArgumentGroup(parser, "model", "model configuration")
model_g.add_arg("word_emb_dim", int, 128,
"The dimension in which a word is embedded.")
model_g.add_arg("grnn_hidden_dim", int, 128,
"The number of hidden nodes in the GRNN layer.")
model_g.add_arg("bigru_num", int, 2,
"The number of bi_gru layers in the network.")
model_g.add_arg("use_cuda", bool, False, "If set, use GPU for training.")
# 2. data parameters
data_g = utils.ArgumentGroup(parser, "data", "data paths")
data_g.add_arg("word_dict_path", str, "./conf/word.dic",
"The path of the word dictionary.")
data_g.add_arg("label_dict_path", str, "./conf/tag.dic",
"The path of the label dictionary.")
data_g.add_arg("word_rep_dict_path", str, "./conf/q2b.dic",
"The path of the word replacement Dictionary.")
data_g.add_arg("test_data", str, "./data/test.tsv",
"The folder where the training data is located.")
data_g.add_arg("save_bin_path", str, "./data/test_eval_1000.bin",
train_feed_list=train_feed_list,
train_fetch_list=train_fetch_list,
eval_program=test_program,
eval_reader=test_reader,
eval_feed_list=test_feed_list,
eval_fetch_list=test_fetch_list,
teacher_programs=[],
train_optimizer=optimizer,
distiller_optimizer=None)
com_pass.config(args.compress_config)
com_pass.run()
if __name__ == "__main__":
# 参数控制可以根据需求使用argparse，yaml或者json
# 对NLP任务推荐使用PALM下定义的configure，可以统一argparse，yaml或者json格式的配置文件。
parser = argparse.ArgumentParser(__doc__)
utils.load_yaml(parser, 'conf/args.yaml')
user_parser = utils.ArgumentGroup(parser, "model", "model configuration")
user_parser.add_arg("compress_config", str, 'conf/quantization.yaml',
"The compress configure file")
args = parser.parse_args()
check_cuda(args.use_cuda)
print(args)
do_compress(args)
