""" @File : load_albert.py @Author : Pengy @Date : 2020/9/28 @Description : Input your description here ... """ from bert4keras.models import build_transformer_model from bert4keras.tokenizers import SpTokenizer from keras.layers import LSTM, Dense from keras.models import Model import numpy as np config_path = '../Models/albert_base_v2/albert_base/albert_config.json' checkpoint_path = '../Models/albert_base_v2/albert_base/model.ckpt-best' vocab_path = '../Models/albert_base_v2/albert_base/30k-clean.vocab' spm_path = '../Models/albert_base_v2/albert_base/30k-clean.model' tokenizer = SpTokenizer(spm_path) model = build_transformer_model(config_path=config_path, checkpoint_path=checkpoint_path, model='albert') model.summary() token_ids, segment_ids = tokenizer.encode('language model') print(model.predict([np.array([token_ids]), np.array([segment_ids])])) output = LSTM(64)(model.output) output = Dense(32)(output) my_model = Model(model.input, output) my_model.summary()
else: i = sp_tokenizer.token_to_id(u'\u2581' + t) if i == 2: i = sp_tokenizer.token_to_id(t) if i != 2: keep_tokens.append(i) new_token_dict[t] = len(new_token_dict) keep_tokens = [2] * 106 + keep_tokens keep_tokens_inv = {j: i for i, j in enumerate(keep_tokens)} compound_tokens = [] for t, _ in sorted(token_dict.items(), key=lambda s: s[1]): if t not in new_token_dict: new_token_dict[t] = len(new_token_dict) ids = [keep_tokens_inv.get(i, 0) for i in sp_tokenizer.encode(t)[0]] compound_tokens.append(ids) save_vocab(dict_path_2, new_token_dict) # 构建分词器 tokenizer = Tokenizer(new_token_dict, do_lower_case=True, pre_tokenize=lambda s: jieba.cut(s, HMM=False)) def corpus(): """语料生成器 """ while True: f = '/root/data_pretrain/data_shuf.json'