def pre(step, prefix):
# configs
max_seqlen = 256
batch_size = 64
num_epochs = 10
lr = 5e-5
weight_decay = 0.01
vocab_path = './pretrain/BERT-zh-base/vocab.txt'
predict_file = './data/xidata/text-0.tsv' # !!!!!!!!!!!!!!!!!!!!!
config = json.load(open('./pretrain/BERT-zh-base/bert_config.json'))
input_dim = config['hidden_size']
num_classes = 3
dropout_prob = 0.1
random_seed = 1
step = step
task_name = 'news'
save_path = './outputs/'
pred_output = prefix + 'predict/'
save_type = 'ckpt'
print_steps = 10000
pre_params = './pretrain/BERT-zh-base/params'
# ----------------------- for prediction -----------------------
# step 1-1: create readers for prediction
#print('prepare to predict...')
predict_cls_reader = palm.reader.ClassifyReader(vocab_path, max_seqlen, seed=random_seed, phase='predict')
# step 1-2: load the training data
predict_cls_reader.load_data(predict_file, batch_size)
# step 2: create a backbone of the model to extract text features
pred_bert = palm.backbone.BERT.from_config(config, phase='predict')
# step 3: register the backbone in reader
predict_cls_reader.register_with(pred_bert)
# step 4: create the task output head
trainer = palm.Trainer(task_name)
cls_pred_head = palm.head.Classify(num_classes, input_dim, phase='predict')
# step 5: build forward graph with backbone and task head
trainer.build_predict_forward(pred_bert, cls_pred_head)
# step 6: load checkpoint
# model_path = './outputs/ckpt.step'+str(save_steps)
model_path = step
#'./outputs/ckpt.step'+str(step)
trainer.load_ckpt(model_path)
# step 7: fit prepared reader and data
trainer.fit_reader(predict_cls_reader, phase='predict')
# step 8: predict
#print('predicting..')
trainer.predict(print_steps=print_steps, output_dir=pred_output)
print('---------'+ str(step) + '----------')
seed=random_seed,
lang='cn')
# step 1-2: load the training data
cls_reader.load_data(train_file, batch_size, num_epochs=num_epochs)
# step 2: create a backbone of the model to extract text features
bert = palm.backbone.BERT.from_config(config)
# step 3: register the backbone in reader
cls_reader.register_with(bert)
# step 4: create the task output head
cls_head = palm.head.Classify(num_classes, input_dim, dropout_prob)
# step 5-1: create a task trainer
trainer = palm.Trainer(task_name)
# step 5-2: build forward graph with backbone and task head
loss_var = trainer.build_forward(bert, cls_head)
# step 6-1*: use warmup
n_steps = cls_reader.num_examples * num_epochs // batch_size
warmup_steps = int(0.1 * n_steps)
sched = palm.lr_sched.TriangularSchedualer(warmup_steps, n_steps)
# step 6-2: create a optimizer
adam = palm.optimizer.Adam(loss_var, lr)
# step 6-3: build backward
trainer.build_backward(optimizer=adam, weight_decay=weight_decay)
# step 7: fit prepared reader and data
trainer.fit_reader(cls_reader)
# step 1-1: create readers for training
match_reader = palm.reader.MatchReader(vocab_path=vocab_path, max_len=max_seqlen, seed=random_seed)
# step 1-2: load the training data
match_reader.load_data(input_file=train_file, file_format='tsv', num_epochs=num_epochs, batch_size=batch_size)
# step 2: create a backbone of the model to extract text features
ernie = palm.backbone.ERNIE.from_config(config=config, phase='train')
# step 3: register the backbone in reader
match_reader.register_with(backbone=ernie)
# step 4: create the task output head
match_head = palm.head.Match(num_classes=num_classes, input_dim=input_dim, dropout_prob=dropout_prob)
# step 5-1: create a task trainer
trainer = palm.Trainer(name=task_name)
# step 5-2: build forward graph with backbone and task head
loss_var = trainer.build_forward(backbone=ernie, task_head=match_head)
# step 6-1*: use warmup
n_steps = match_reader.num_examples * num_epochs // batch_size
save_steps = match_reader.num_examples // batch_size
# n_steps: 200055 save_steps: 66685
print('n_steps: {} save_steps: {}'.format(n_steps, save_steps), flush=True)
warmup_steps = int(0.1 * n_steps)
sched = palm.lr_sched.TriangularSchedualer(warmup_steps=warmup_steps, num_train_steps=n_steps)
# step 6-2: create a optimizer
adam = palm.optimizer.Adam(loss_var=loss_var, lr=lr, lr_schedualer=sched)
# step 6-3: build backward
trainer.build_backward(optimizer=adam, weight_decay=weight_decay)
cls_reader2.register_with(ernie)
print(cls_reader.outputs_attr)
print("preparing data...")
print(cls_reader.num_examples)
cls_reader.load_data(train_file, batch_size)
cls_reader2.load_data(train_file, batch_size)
print(cls_reader.num_examples)
print('done!')
# 创建任务头（task head），如分类、匹配、机器阅读理解等。每个任务头有跟该任务相关的必选/可选参数。注意，任务头与reader是解耦合的，只要任务头依赖的数据集侧的字段能被reader提供，那么就是合法的
cls_head = palm.head.Classify(4, 1024, 0.1)
cls_head2 = palm.head.Classify(4, 1024, 0.1)
# 根据reader和任务头来创建一个训练器trainer，trainer代表了一个训练任务，内部维护着训练进程、和任务的关键信息，并完成合法性校验，该任务的模型保存、载入等相关规则控制
trainer = palm.Trainer('cls')
trainer2 = palm.Trainer('senti_cls')
mh_trainer = palm.MultiHeadTrainer([trainer, trainer2])
# match4mrqa.reuse_head_with(mrc4mrqa)
# data_vars = cls_reader.build()
# output_vars = ernie.build(data_vars)
# cls_head.build({'backbone': output_vars, 'reader': data_vars})
loss_var = mh_trainer.build_forward(ernie, [cls_head, cls_head2])
n_steps = cls_reader.num_examples * num_epochs // batch_size
warmup_steps = int(0.1 * n_steps)
print(warmup_steps)
sched = palm.lr_sched.TriangularSchedualer(warmup_steps, n_steps)
cls_reader.load_data(train_intent, batch_size=batch_size, num_epochs=None)
# step 2: create a backbone of the model to extract text features
ernie = palm.backbone.ERNIE.from_config(config)
# step 3: register readers with ernie backbone
seq_label_reader.register_with(ernie)
cls_reader.register_with(ernie)
# step 4: create task output heads
seq_label_head = palm.head.SequenceLabel(num_classes, input_dim,
dropout_prob)
cls_head = palm.head.Classify(num_classes_intent, input_dim, dropout_prob)
# step 5-1: create task trainers and multiHeadTrainer
trainer_seq_label = palm.Trainer("slot", mix_ratio=1.0)
trainer_cls = palm.Trainer("intent", mix_ratio=1.0)
trainer = palm.MultiHeadTrainer([trainer_seq_label, trainer_cls])
# # step 5-2: build forward graph with backbone and task head
loss1 = trainer_cls.build_forward(ernie, cls_head)
loss2 = trainer_seq_label.build_forward(ernie, seq_label_head)
loss_var = trainer.build_forward()
# step 6-1*: enable warmup for better fine-tuning
n_steps = seq_label_reader.num_examples * 1.5 * num_epochs // batch_size
warmup_steps = int(0.1 * n_steps)
sched = palm.lr_sched.TriangularSchedualer(warmup_steps, n_steps)
# step 6-2: build a optimizer
adam = palm.optimizer.Adam(loss_var, lr, sched)
# step 6-3: build backward graph
trainer.build_backward(optimizer=adam, weight_decay=weight_decay)
slot_reader.load_data(train_slot, file_format='tsv', num_epochs=None, batch_size=batch_size)
intent_reader.load_data(train_intent, batch_size=batch_size, num_epochs=None)
# step 2: create a backbone of the model to extract text features
ernie = palm.backbone.ERNIE.from_config(config, phase='predict')
# step 3: register readers with ernie backbone
slot_reader.register_with(ernie)
intent_reader.register_with(ernie)
# step 4: create task output heads
slot_head = palm.head.SequenceLabel(num_classes, input_dim, dropout_prob, phase='predict')
intent_head = palm.head.Classify(num_classes_intent, input_dim, dropout_prob, phase='predict')
# step 5-1: create task trainers and multiHeadTrainer
trainer_slot = palm.Trainer("slot", mix_ratio=1.0)
trainer_intent = palm.Trainer("intent", mix_ratio=1.0)
trainer = palm.MultiHeadTrainer([trainer_slot, trainer_intent])
# # step 5-2: build forward graph with backbone and task head
vars = trainer_intent.build_predict_forward(ernie, intent_head)
vars = trainer_slot.build_predict_forward(ernie, slot_head)
loss_var = trainer.build_predict_forward()
# load checkpoint
trainer.load_ckpt('outputs/ckpt.step300')
# merge inference readers
joint_iterator = trainer.merge_inference_readers([slot_reader, intent_reader])
# for test
# batch = next(joint_iterator('slot'))
vocab_path, max_seqlen, label_map, seed=random_seed, phase='predict')
# step 1-2: load the training data
predict_seq_label_reader.load_data(predict_file, batch_size)
# step 2: create a backbone of the model to extract text features
pred_ernie = palm.backbone.ERNIE.from_config(config, phase='predict')
# step 3: register the backbone in reader
predict_seq_label_reader.register_with(pred_ernie)
# step 4: create the task output head
seq_label_pred_head = palm.head.SequenceLabel(num_classes,
input_dim,
phase='predict')
# step 5-1: create a task trainer
trainer_seq_label = palm.Trainer("slot")
# step 5-2: build forward graph with backbone and task head
trainer_seq_label.build_predict_forward(pred_ernie, seq_label_pred_head)
# step 6: load checkpoint
pred_model_path = './outputs/ckpt.step4641'
trainer_seq_label.load_ckpt(pred_model_path)
# step 7: fit prepared reader and data
trainer_seq_label.fit_reader(predict_seq_label_reader, phase='predict')
# step 8: predict
print('predicting..')
trainer_seq_label.predict(print_steps=print_steps, output_dir=pred_output)
max_seqlen,
seed=random_seed,
phase='predict')
# step 1-2: load the training data
predict_cls_reader.load_data(predict_file, batch_size)
# step 2: create a backbone of the model to extract text features
pred_ernie = palm.backbone.ERNIE.from_config(config, phase='predict')
# step 3: register the backbone in reader
predict_cls_reader.register_with(pred_ernie)
# step 4: create the task output head
cls_pred_head = palm.head.Classify(num_classes, input_dim, phase='predict')
# step 5-1: create a task trainer
trainer = palm.Trainer("intent")
# step 5-2: build forward graph with backbone and task head
trainer.build_predict_forward(pred_ernie, cls_pred_head)
# step 6: load checkpoint
pred_model_path = './outputs/ckpt.step4641'
trainer.load_ckpt(pred_model_path)
# step 7: fit prepared reader and data
trainer.fit_reader(predict_cls_reader, phase='predict')
# step 8: predict
print('predicting..')
trainer.predict(print_steps=print_steps, output_dir=pred_output)
