def pre(step, prefix): # configs max_seqlen = 256 batch_size = 64 num_epochs = 10 lr = 5e-5 weight_decay = 0.01 vocab_path = './pretrain/BERT-zh-base/vocab.txt' predict_file = './data/xidata/text-0.tsv' # !!!!!!!!!!!!!!!!!!!!! config = json.load(open('./pretrain/BERT-zh-base/bert_config.json')) input_dim = config['hidden_size'] num_classes = 3 dropout_prob = 0.1 random_seed = 1 step = step task_name = 'news' save_path = './outputs/' pred_output = prefix + 'predict/' save_type = 'ckpt' print_steps = 10000 pre_params = './pretrain/BERT-zh-base/params' # ----------------------- for prediction ----------------------- # step 1-1: create readers for prediction #print('prepare to predict...') predict_cls_reader = palm.reader.ClassifyReader(vocab_path, max_seqlen, seed=random_seed, phase='predict') # step 1-2: load the training data predict_cls_reader.load_data(predict_file, batch_size) # step 2: create a backbone of the model to extract text features pred_bert = palm.backbone.BERT.from_config(config, phase='predict') # step 3: register the backbone in reader predict_cls_reader.register_with(pred_bert) # step 4: create the task output head trainer = palm.Trainer(task_name) cls_pred_head = palm.head.Classify(num_classes, input_dim, phase='predict') # step 5: build forward graph with backbone and task head trainer.build_predict_forward(pred_bert, cls_pred_head) # step 6: load checkpoint # model_path = './outputs/ckpt.step'+str(save_steps) model_path = step #'./outputs/ckpt.step'+str(step) trainer.load_ckpt(model_path) # step 7: fit prepared reader and data trainer.fit_reader(predict_cls_reader, phase='predict') # step 8: predict #print('predicting..') trainer.predict(print_steps=print_steps, output_dir=pred_output) print('---------'+ str(step) + '----------')
seed=random_seed, lang='cn') # step 1-2: load the training data cls_reader.load_data(train_file, batch_size, num_epochs=num_epochs) # step 2: create a backbone of the model to extract text features bert = palm.backbone.BERT.from_config(config) # step 3: register the backbone in reader cls_reader.register_with(bert) # step 4: create the task output head cls_head = palm.head.Classify(num_classes, input_dim, dropout_prob) # step 5-1: create a task trainer trainer = palm.Trainer(task_name) # step 5-2: build forward graph with backbone and task head loss_var = trainer.build_forward(bert, cls_head) # step 6-1*: use warmup n_steps = cls_reader.num_examples * num_epochs // batch_size warmup_steps = int(0.1 * n_steps) sched = palm.lr_sched.TriangularSchedualer(warmup_steps, n_steps) # step 6-2: create a optimizer adam = palm.optimizer.Adam(loss_var, lr) # step 6-3: build backward trainer.build_backward(optimizer=adam, weight_decay=weight_decay) # step 7: fit prepared reader and data trainer.fit_reader(cls_reader)
# step 1-1: create readers for training match_reader = palm.reader.MatchReader(vocab_path=vocab_path, max_len=max_seqlen, seed=random_seed) # step 1-2: load the training data match_reader.load_data(input_file=train_file, file_format='tsv', num_epochs=num_epochs, batch_size=batch_size) # step 2: create a backbone of the model to extract text features ernie = palm.backbone.ERNIE.from_config(config=config, phase='train') # step 3: register the backbone in reader match_reader.register_with(backbone=ernie) # step 4: create the task output head match_head = palm.head.Match(num_classes=num_classes, input_dim=input_dim, dropout_prob=dropout_prob) # step 5-1: create a task trainer trainer = palm.Trainer(name=task_name) # step 5-2: build forward graph with backbone and task head loss_var = trainer.build_forward(backbone=ernie, task_head=match_head) # step 6-1*: use warmup n_steps = match_reader.num_examples * num_epochs // batch_size save_steps = match_reader.num_examples // batch_size # n_steps: 200055 save_steps: 66685 print('n_steps: {} save_steps: {}'.format(n_steps, save_steps), flush=True) warmup_steps = int(0.1 * n_steps) sched = palm.lr_sched.TriangularSchedualer(warmup_steps=warmup_steps, num_train_steps=n_steps) # step 6-2: create a optimizer adam = palm.optimizer.Adam(loss_var=loss_var, lr=lr, lr_schedualer=sched) # step 6-3: build backward trainer.build_backward(optimizer=adam, weight_decay=weight_decay)
cls_reader2.register_with(ernie) print(cls_reader.outputs_attr) print("preparing data...") print(cls_reader.num_examples) cls_reader.load_data(train_file, batch_size) cls_reader2.load_data(train_file, batch_size) print(cls_reader.num_examples) print('done!') # 创建任务头(task head),如分类、匹配、机器阅读理解等。每个任务头有跟该任务相关的必选/可选参数。注意,任务头与reader是解耦合的,只要任务头依赖的数据集侧的字段能被reader提供,那么就是合法的 cls_head = palm.head.Classify(4, 1024, 0.1) cls_head2 = palm.head.Classify(4, 1024, 0.1) # 根据reader和任务头来创建一个训练器trainer,trainer代表了一个训练任务,内部维护着训练进程、和任务的关键信息,并完成合法性校验,该任务的模型保存、载入等相关规则控制 trainer = palm.Trainer('cls') trainer2 = palm.Trainer('senti_cls') mh_trainer = palm.MultiHeadTrainer([trainer, trainer2]) # match4mrqa.reuse_head_with(mrc4mrqa) # data_vars = cls_reader.build() # output_vars = ernie.build(data_vars) # cls_head.build({'backbone': output_vars, 'reader': data_vars}) loss_var = mh_trainer.build_forward(ernie, [cls_head, cls_head2]) n_steps = cls_reader.num_examples * num_epochs // batch_size warmup_steps = int(0.1 * n_steps) print(warmup_steps) sched = palm.lr_sched.TriangularSchedualer(warmup_steps, n_steps)
cls_reader.load_data(train_intent, batch_size=batch_size, num_epochs=None) # step 2: create a backbone of the model to extract text features ernie = palm.backbone.ERNIE.from_config(config) # step 3: register readers with ernie backbone seq_label_reader.register_with(ernie) cls_reader.register_with(ernie) # step 4: create task output heads seq_label_head = palm.head.SequenceLabel(num_classes, input_dim, dropout_prob) cls_head = palm.head.Classify(num_classes_intent, input_dim, dropout_prob) # step 5-1: create task trainers and multiHeadTrainer trainer_seq_label = palm.Trainer("slot", mix_ratio=1.0) trainer_cls = palm.Trainer("intent", mix_ratio=1.0) trainer = palm.MultiHeadTrainer([trainer_seq_label, trainer_cls]) # # step 5-2: build forward graph with backbone and task head loss1 = trainer_cls.build_forward(ernie, cls_head) loss2 = trainer_seq_label.build_forward(ernie, seq_label_head) loss_var = trainer.build_forward() # step 6-1*: enable warmup for better fine-tuning n_steps = seq_label_reader.num_examples * 1.5 * num_epochs // batch_size warmup_steps = int(0.1 * n_steps) sched = palm.lr_sched.TriangularSchedualer(warmup_steps, n_steps) # step 6-2: build a optimizer adam = palm.optimizer.Adam(loss_var, lr, sched) # step 6-3: build backward graph trainer.build_backward(optimizer=adam, weight_decay=weight_decay)
slot_reader.load_data(train_slot, file_format='tsv', num_epochs=None, batch_size=batch_size) intent_reader.load_data(train_intent, batch_size=batch_size, num_epochs=None) # step 2: create a backbone of the model to extract text features ernie = palm.backbone.ERNIE.from_config(config, phase='predict') # step 3: register readers with ernie backbone slot_reader.register_with(ernie) intent_reader.register_with(ernie) # step 4: create task output heads slot_head = palm.head.SequenceLabel(num_classes, input_dim, dropout_prob, phase='predict') intent_head = palm.head.Classify(num_classes_intent, input_dim, dropout_prob, phase='predict') # step 5-1: create task trainers and multiHeadTrainer trainer_slot = palm.Trainer("slot", mix_ratio=1.0) trainer_intent = palm.Trainer("intent", mix_ratio=1.0) trainer = palm.MultiHeadTrainer([trainer_slot, trainer_intent]) # # step 5-2: build forward graph with backbone and task head vars = trainer_intent.build_predict_forward(ernie, intent_head) vars = trainer_slot.build_predict_forward(ernie, slot_head) loss_var = trainer.build_predict_forward() # load checkpoint trainer.load_ckpt('outputs/ckpt.step300') # merge inference readers joint_iterator = trainer.merge_inference_readers([slot_reader, intent_reader]) # for test # batch = next(joint_iterator('slot'))
vocab_path, max_seqlen, label_map, seed=random_seed, phase='predict') # step 1-2: load the training data predict_seq_label_reader.load_data(predict_file, batch_size) # step 2: create a backbone of the model to extract text features pred_ernie = palm.backbone.ERNIE.from_config(config, phase='predict') # step 3: register the backbone in reader predict_seq_label_reader.register_with(pred_ernie) # step 4: create the task output head seq_label_pred_head = palm.head.SequenceLabel(num_classes, input_dim, phase='predict') # step 5-1: create a task trainer trainer_seq_label = palm.Trainer("slot") # step 5-2: build forward graph with backbone and task head trainer_seq_label.build_predict_forward(pred_ernie, seq_label_pred_head) # step 6: load checkpoint pred_model_path = './outputs/ckpt.step4641' trainer_seq_label.load_ckpt(pred_model_path) # step 7: fit prepared reader and data trainer_seq_label.fit_reader(predict_seq_label_reader, phase='predict') # step 8: predict print('predicting..') trainer_seq_label.predict(print_steps=print_steps, output_dir=pred_output)
max_seqlen, seed=random_seed, phase='predict') # step 1-2: load the training data predict_cls_reader.load_data(predict_file, batch_size) # step 2: create a backbone of the model to extract text features pred_ernie = palm.backbone.ERNIE.from_config(config, phase='predict') # step 3: register the backbone in reader predict_cls_reader.register_with(pred_ernie) # step 4: create the task output head cls_pred_head = palm.head.Classify(num_classes, input_dim, phase='predict') # step 5-1: create a task trainer trainer = palm.Trainer("intent") # step 5-2: build forward graph with backbone and task head trainer.build_predict_forward(pred_ernie, cls_pred_head) # step 6: load checkpoint pred_model_path = './outputs/ckpt.step4641' trainer.load_ckpt(pred_model_path) # step 7: fit prepared reader and data trainer.fit_reader(predict_cls_reader, phase='predict') # step 8: predict print('predicting..') trainer.predict(print_steps=print_steps, output_dir=pred_output)