def test_process(self): hparam = Hparams() hparam.load_from_config_file('/search/odin/yyk/workspace/AiSpace/configs/custom/idiom_generator.yml') hparam.stand_by() hparam.cascade_set("model_load_path", "/search/odin/yyk/workspace/AiSpace/save/test_bert_for_text_generation_idiom__idiom_generator_119_23") model, (losses, loss_weights), metrics, optimizer = build_model(hparam) model.compile(optimizer=optimizer, loss=losses, metrics=metrics) tokenizer = CPMTokenizer(hparam.dataset.tokenizer) input = "春眠不觉晓" input_tokens = tokenizer.tokenize(input) + [tokenizer.vocab.sep_token] input_encoded = tokenizer.encode(input_tokens) input_ids = tf.constant([input_encoded['input_ids']], dtype=tf.int32) attention_mask = tf.constant([[1] * len(input_encoded['input_ids'])], dtype=tf.int32) input_dict = { "input_ids": input_ids, "attention_mask": attention_mask } # output = model(input_dict) output = model.generate(input_ids, **hparam.generation_attributes) print(input_encoded) output = tokenizer.decode(output.numpy().reshape([-1]).tolist()) print(output)
def test_gpt2_checkpoint(self): hparam = Hparams() hparam.load_from_config_file('/search/odin/yyk/workspace/AiSpace/configs/custom/test_gpt2.yml') # hparam.load_from_config_file('/search/data1/yyk/workspace/AiSpace/configs/glue_zh/tnews.yml') # hparam.load_from_config_file('/search/data1/yyk/workspace/AiSpace/configs/glue_zh/cmrc2018.yml') hparam.stand_by() model_path = "/search/odin/yyk/data/pretrained/gpt/cpm-lm-tf2_v2" # model1 = tf.keras.models.load_model(model_path) # model_gold = model1.trainable_variables # ckpt_vars = [itm for itm in tf.train.list_variables(ckpt) if itm[0].find('adam') == -1] # ckpt_vars = [itm for itm in tf.train.list_variables(hparam.pretrained.model_path) if itm[0].find('adam') == -1] model, (losses, loss_weights), metrics, optimizer = build_model(hparam) model.compile(optimizer=optimizer, loss=losses, metrics=metrics) # model_vars = model.trainable_variables for itm in model_vars: print(f"{itm.name}, {itm.shape}") # print(itm.numpy()) # print(type(itm.numpy())) # break print()
def test_eval(self): hparams = Hparams() hparams.load_from_config_file("../../configs/glue_zh/tnews_k_fold.yml") hparams.stand_by() ckpts = [ "../../save/test_textcnn_for_classification_119_14/k_fold/1/model_saved/model", "../../save/test_textcnn_for_classification_119_14/k_fold/2/model_saved/model", ] evaluation(hparams, checkpoints=ckpts)
def test_process(self): hparam = Hparams() hparam.load_from_config_file( '/search/odin/yyk/workspace/AiSpace/configs/custom/test_gpt2.yml') hparam.stand_by() model, (losses, loss_weights), metrics, optimizer = build_model(hparam) model.compile(optimizer=optimizer, loss=losses, metrics=metrics) model_vars = model.trainable_variables model_path = "/search/odin/yyk/data/pretrained/gpt/cpm-lm-tf2_v2" tf_huggingface_gpt2_adapter(model_vars, model_path)
def test_init(self): hparams = Hparams() hparams.load_from_config_file("../../../configs/custom/test_gpt2.yml") hparams.stand_by() tokenizer = CPMTokenizer(hparams.dataset.tokenizer) a = "这两天,XLNet貌似也引起了NLP圈的极大关注,从实验数据看,在某些场景下,确实XLNet相对Bert有很大幅度的提升。" b = "就像我们之前说的,感觉Bert打开两阶段模式的魔法盒开关后,在这条路上,会有越来越多的同行者,而XLNet就是其中比较引人注目的一位" res = tokenizer.encode(a, b) print(res)
def test_electra_checkpoint(self): hparam = Hparams() hparam.load_from_config_file('/search/data1/yyk/workspace/AiSpace/configs/glue_zh/tnews.yml') hparam.stand_by() # ckpt = "/search/data1/yyk/workspace/projects/ERNIE/ernie/checkpoints" # ckpt_vars = [itm for itm in tf.train.list_variables(ckpt) if itm[0].find('adam') == -1] ckpt_vars = [itm for itm in tf.train.list_variables(hparam.pretrained.model_path) if itm[0].find('adam') == -1 and not itm[0].endswith("lamb_m") and not itm[0].endswith("lamb_v")] model, (losses, loss_weights), metrics, optimizer = build_model(hparam) model.compile(optimizer=optimizer, loss=losses, metrics=metrics) model_vars = model.trainable_variables print()
def test_lstc_load(self): hparams = Hparams() hparams.load_from_config_file("../configs/custom/test_gpt2.yml") hparams.stand_by() checksum_dir = "../aispace/datasets/url_checksums" tfds.download.add_checksums_dir(checksum_dir) # download_config = DownloadConfig(register_checksums=True) tnews = tfds.load( "idiom/idiom_generator", # data_dir="/search/data1/yyk/data/datasets/glue_zh", split="train[90%:]", data_dir="../data", builder_kwargs={'hparams': hparams}, # download_and_prepare_kwargs={'download_config': download_config} ) tokenizer = BertTokenizer(hparams.dataset.tokenizer) # id_to_label = {v: k for k, v in hparams.duee_event_type_labels.items()} label_counter = {} i = 0 for itm in tnews: # for k, v in itm.items(): # if v.shape[0] == 151: # print(itm) # break # print(itm) # print() # print(tokenizer.decode([int(t) for t in itm["input_ids"].numpy().tolist()])) # break i += 1 # l = hparams.dataset.outputs[0].labels[tf.argmax(itm["output_1"], -1).numpy().tolist()] # print(id_to_label[l]) # if id_to_label[l] not in label_counter: # label_counter[id_to_label[l]] = 0 # label_counter[id_to_label[l]] += 1 # print(label_counter) # print(len(label_counter)) print(i) # python -u aispace/trainer.py \ # --experiment_name test \ # --model_name bert_for_classification \ # --schedule train_and_eval \ # --config_name tnews \ # --config_dir ./configs/glue_zh \ # --gpus 0 1 2 3
def test_lstc_load(self): hparams = Hparams() hparams.load_from_config_file( "../configs/2020_LSTC/DuEE_keyphrase.yml") hparams.stand_by() checksum_dir = "../aispace/datasets/url_checksums" tfds.download.add_checksums_dir(checksum_dir) # download_config = DownloadConfig(register_checksums=True) tnews = tfds.load( "lstc_2020/DuEE_role", # data_dir="/search/data1/yyk/data/datasets/glue_zh", data_dir="../data", builder_kwargs={'hparams': hparams}, # download_and_prepare_kwargs={'download_config': download_config} ) # tokenizer = BertTokenizer(hparams.dataset.tokenizer) # s = "BCI下架新疆棉花产品" # res = tokenizer.tokenize(s, True) # print(res) # id_to_label = {v: k for k, v in hparams.duee_event_type_labels.items()} label_counter = {} for itm in tnews["train"]: # for k, v in itm.items(): # if v.shape[0] == 151: # print(itm) # break print(itm) print() # print(tokenizer.decode([int(t) for t in itm["input_ids"].numpy().tolist()])) break # l = hparams.dataset.outputs[0].labels[tf.argmax(itm["output_1"], -1).numpy().tolist()] # print(id_to_label[l]) # if id_to_label[l] not in label_counter: # label_counter[id_to_label[l]] = 0 # label_counter[id_to_label[l]] += 1 print(label_counter) print(len(label_counter)) # python -u aispace/trainer.py \ # --experiment_name test \ # --model_name bert_for_classification \ # --schedule train_and_eval \ # --config_name tnews \ # --config_dir ./configs/glue_zh \ # --gpus 0 1 2 3
def test_glue_load(self): hparams = Hparams() hparams.load_from_config_file("../configs/qa/dureader_yesno.yml") hparams.stand_by() checksum_dir = "../aispace/datasets/url_checksums" tfds.download.add_checksums_dir(checksum_dir) download_config = DownloadConfig(register_checksums=True) print(tfds.list_builders()) dureader = tfds.load( "dureader/yesno", # data_dir="/search/data1/yyk/data/datasets/glue_zh", data_dir="../data/dureader", builder_kwargs={'hparams': hparams}, download_and_prepare_kwargs={'download_config': download_config}) for itm in dureader['train']: print(itm) break print() # train_dataset, dev_dataset, dataset_info = next(load_dataset(hparams, ret_test=False)) # test_dataset = next(load_dataset(hparams, ret_train=True, ret_dev=True, ret_test=True, ret_info=True))[0] # total, zero = 0, 0 # for itm in tqdm(test_dataset): # tt = itm[0]['input_ids'].numpy().tolist() # print(itm[0]['p_mask'].numpy().tolist()) # print(itm[0]['start_position'].numpy().tolist()) # print(itm[0]['end_position'].numpy().tolist()) # print(tt) # break # total += 1 # zero += len([t for t in tt if t == 0]) # print() # print(f"{zero}, {total}, {zero / float(total)}") # print(total) # python -u aispace/trainer.py \ # --experiment_name test \ # --model_name bert_for_classification \ # --schedule train_and_eval \ # --config_name tnews \ # --config_dir ./configs/glue_zh \ # --gpus 0 1 2 3
def test_glue_load(self): hparams = Hparams() hparams.load_from_config_file("../configs/glue_zh/cmrc2018.yml") hparams.stand_by() # checksum_dir = "../aispace/datasets/url_checksums" # tfds.download.add_checksums_dir(checksum_dir) # download_config = DownloadConfig(register_checksums=True) # cmrc2018 = tfds.load("glue_zh/cmrc2018", # # data_dir="/search/data1/yyk/data/datasets/glue_zh", # data_dir="../data/glue_zh", # builder_kwargs={'hparams': hparams}, # download_and_prepare_kwargs={'download_config': download_config} # ) # train_dataset, dev_dataset, dataset_info = next(load_dataset(hparams, ret_test=False)) test_dataset = next( load_dataset(hparams, ret_train=False, ret_dev=True, ret_test=False, ret_info=False))[0] total, zero = 0, 0 for itm in test_dataset: tt = itm[0]['start_position'].numpy().tolist() # print(itm[0]['p_mask'].numpy().tolist()) # print(itm[0]['start_position'].numpy().tolist()) # print(itm[0]['end_position'].numpy().tolist()) # break total += len(tt) zero += len([t for t in tt if t == 0]) print() print(f"{zero}, {total}, {zero / float(total)}") # python -u aispace/trainer.py \ # --experiment_name test \ # --model_name bert_for_classification \ # --schedule train_and_eval \ # --config_name tnews \ # --config_dir ./configs/glue_zh \ # --gpus 0 1 2 3
def test_electra_checkpoint(self): hparam = Hparams() hparam.load_from_config_file( '/search/data1/yyk/workspace/AiSpace/configs/glue_zh/tnews.yml') # hparam.load_from_config_file('/search/data1/yyk/workspace/AiSpace/configs/glue_zh/cmrc2018.yml') hparam.stand_by() # ckpt = "/search/data1/yyk/workspace/projects/ERNIE/ernie/checkpoints" ckpt = "/search/data1/yyk/data/pretrained/albert/albert_large_zh_google/model.ckpt-best" ckpt_vars = [ itm for itm in tf.train.list_variables(ckpt) if itm[0].find('adam') == -1 ] # ckpt_vars = [itm for itm in tf.train.list_variables(hparam.pretrained.model_path) if itm[0].find('adam') == -1] model, (losses, loss_weights), metrics, optimizer = build_model(hparam) model.compile(optimizer=optimizer, loss=losses, metrics=metrics) model_vars = model.trainable_variables print()
def evaluation(hparams: Hparams, checkpoints=None, model=None, test_dataset=None): """Evaluate the model and build report according to different task. :param model: :param test_dataset: :param hparams: :return: """ logger.info("Start Evaluate.") output_hparams = deepcopy(hparams.dataset.outputs) if test_dataset is None: test_dataset = next( load_dataset(hparams, ret_train=False, ret_dev=False, ret_info=False))[0] if model is None: # build model (model, ) = build_model(hparams, return_losses=False, return_metrics=False, return_optimizer=False) # predict using default model saved if checkpoints is None: # load weights if not os.path.exists(hparams.get_model_filename() + ".index"): logger.warning( f"Model from {hparams.get_model_filename()} is not exists, load nothing!" ) else: logger.info( f"Load model weights from {hparams.get_model_filename()}") model.load_weights(hparams.get_model_filename()) # prediction # print(model.evaluate(test_dataset)) for inputs, outputs in tqdm(test_dataset): model_outputs = model.predict(inputs) if not isinstance(model_outputs, (tuple, list)): model_outputs = (model_outputs, ) for idx, one_output_hparam in enumerate(output_hparams): if "ground_truth" not in one_output_hparam: one_output_hparam["ground_truth"] = [] if "predictions" not in one_output_hparam: one_output_hparam['predictions'] = [] prediction_output = tf.nn.softmax(model_outputs[idx], -1) tmp_name = one_output_hparam.name tmp_type = one_output_hparam.type tmp_ground_truth = outputs[tmp_name] if tmp_type in [CLASSLABEL, LIST_OF_CLASSLABEL, LIST_OF_INT]: if tmp_type in [LIST_OF_INT]: tmp_tg = tf.argmax(tmp_ground_truth, -1) else: tmp_tg = tmp_ground_truth if one_output_hparam.task == NER: # [[sent1], [sent2]] one_output_hparam.ground_truth.extend( tmp_tg.numpy().tolist()) tmp_predictions = tf.argmax(prediction_output, -1).numpy().tolist() one_output_hparam.predictions.extend(tmp_predictions) else: # [1, 0, 1, ...] one_output_hparam.ground_truth.extend( tmp_tg.numpy().reshape(-1).tolist()) tmp_predictions = tf.argmax( prediction_output, -1).numpy().reshape(-1).tolist() one_output_hparam.predictions.extend(tmp_predictions) elif isinstance(checkpoints, (tuple, list)): # predict using multi checkpints from k-fold cross validation. for i, ckpt in enumerate(checkpoints): if not os.path.exists(ckpt + ".index"): logger.warning( f"Model from {ckpt} is not exists, load nothing!") continue else: logger.info(f"Load model weights from {ckpt}") model.load_weights(ckpt) for j, (inputs, outputs) in tqdm(enumerate(test_dataset)): model_outputs = model.predict(inputs) if not isinstance(model_outputs, (tuple, list)): model_outputs = (model_outputs, ) for idx, one_output_hparam in enumerate(output_hparams): prediction_output = tf.nn.softmax(model_outputs[idx], -1) if i == 0: if "ground_truth" not in one_output_hparam: one_output_hparam["ground_truth"] = [] if "predictions" not in one_output_hparam: one_output_hparam['predictions'] = [] one_output_hparam['tmp_preds'] = [] one_output_hparam['tmp_preds'].append( prediction_output) tmp_name = one_output_hparam.name tmp_type = one_output_hparam.type tmp_ground_truth = outputs[tmp_name] if tmp_type in [ CLASSLABEL, LIST_OF_CLASSLABEL, LIST_OF_INT ]: if tmp_type in [LIST_OF_INT]: tmp_tg = tf.argmax(tmp_ground_truth, -1) else: tmp_tg = tmp_ground_truth if one_output_hparam.task == NER: # [[sent1], [sent2]] one_output_hparam.ground_truth.extend( tmp_tg.numpy().tolist()) else: # [1, 0, 1, ...] one_output_hparam.ground_truth.extend( tmp_tg.numpy().reshape(-1).tolist()) else: one_output_hparam['tmp_preds'][j] += prediction_output for idx, one_output_hparam in enumerate(output_hparams): prediction_output = one_output_hparam['tmp_preds'][idx] tmp_type = one_output_hparam.type if tmp_type in [CLASSLABEL, LIST_OF_CLASSLABEL, LIST_OF_INT]: if one_output_hparam.task == NER: # [[sent1], [sent2]] tmp_predictions = tf.argmax(prediction_output, -1).numpy().tolist() one_output_hparam.predictions.extend(tmp_predictions) else: # [1, 0, 1, ...] tmp_predictions = tf.argmax( prediction_output, -1).numpy().reshape(-1).tolist() one_output_hparam.predictions.extend(tmp_predictions) # save reports report_folder = hparams.get_report_dir() # evaluation, TODO more reports for one_output_hparam in output_hparams: ground_truth = one_output_hparam.ground_truth predictions = one_output_hparam.predictions if one_output_hparam.type in [ CLASSLABEL, LIST_OF_CLASSLABEL, LIST_OF_INT ]: # some filename cur_report_folder = os.path.join( report_folder, f'{one_output_hparam.name}_{one_output_hparam.type.lower()}') if not os.path.exists(cur_report_folder): os.makedirs(cur_report_folder) if one_output_hparam.task == NER: labels = one_output_hparam.labels # confusion matrix cm = ConfusionMatrix(_2d_to_1d_list(ground_truth), _2d_to_1d_list(predictions), labels) # ner evaluation labels = list( set([ itm[2:] for itm in labels if itm.startswith("B-") or itm.startswith("I-") ])) ner_eval = NEREvaluator( _id_to_label(ground_truth, one_output_hparam.labels), _id_to_label(predictions, one_output_hparam.labels), labels) ner_results, ner_results_agg = ner_eval.evaluate() save_json(os.path.join(cur_report_folder, "ner_results.json"), ner_results) save_json( os.path.join(cur_report_folder, "ner_results_agg.json"), ner_results_agg) else: cm = ConfusionMatrix(ground_truth, predictions, one_output_hparam.labels) # print some reports print_boxed(f"{one_output_hparam.name} Evaluation") cms = cm.confusion_matrix_visual() if len(cm.label2idx) < 10: print(cms) # save reports to files with open( os.path.join(cur_report_folder, "confusion_matrix.txt"), 'w') as f: f.write(cms) print() print(json.dumps(cm.stats(), indent=4)) save_json(os.path.join(cur_report_folder, "stats.json"), cm.stats()) save_json(os.path.join(cur_report_folder, 'per_class_stats.json'), cm.per_class_stats()) # save reports to hparams hparams['performance'] = Hparams() hparams.performance["stats"] = cm.stats() hparams.performance["per_class_stats"] = cm.per_class_stats() logger.info( f"Save {one_output_hparam.name} reports in {cur_report_folder}" ) else: logger.warning( f"{one_output_hparam.name}'s evaluation has not be implemented." )
def test_dataset_split(self): hparams = Hparams() hparams.load_from_config_file("../configs/glue_zh/tnews.yml") hparams.stand_by() k_fold_experiment(hparams)