def __init__(self, cores=cpu_cores): super(AutoMasterDataHandler, self).__init__() self.cores = cores self.wvtool = WVTool(ndim=config.emb_dim, cores=self.cores, epochs=config.wv_epochs) self.logger = init_logger() self.cols = {'X': ['权利要求'], 'Y': ['正例文本']} self.cols['X+Y'] = self.cols['X'] + self.cols['Y'] self.max_len = {'X': None, 'Y': None} self.df_all, self.df_train, self.df_eval, self.df_test, self.df_merged = pd.DataFrame( columns=['专利序号', '权利要求', '正例文本']), pd.DataFrame( columns=['专利序号', '权利要求', '正例文本']), pd.DataFrame( columns=['专利序号', '权利要求', '正例文本']), pd.DataFrame( columns=['专利序号', '权利要求', '正例文本']), pd.DataFrame( columns=['专利序号', '权利要求', '正例文本']) self.path_userdict = None self.path_stopwords = None self.swtool = None self.path_data_train, self.path_data_eval, self.path_data_test = None, None, None self.path_seg_train, self.path_seg_test, self.path_seg_eval, self.path_seg_merged = None, None, None, None self.path_pad_train_X, self.path_pad_test_X, self.path_pad_train_Y = None, None, None self.path_wv_model, self.path_embedding_matrix = None, None self.path_vocab, self.path_vocab_w2i, self.path_vocab_i2w = None, None, None self.initialize_params()
def __init__(self, ndim=512, cores=1, epochs=10): super(WVTool, self).__init__() self.ndim = ndim self.cores = cores self.epochs = epochs self.model, self.vocab = None, None self.vocab_w2i, self.vocab_i2w = None, None self.embedding_matrix = None self.logger = init_logger() self.labels = { 'PAD': CustomToken(name='PAD', word='<PAD>', index=-1), 'UNK': CustomToken(name='UNK', word='<UNK>', index=-1), 'BOS': CustomToken(name='BOS', word='<BOS>', index=-1), 'EOS': CustomToken(name='EOS', word='<EOS>', index=-1) }
#!/usr/bin/env python3 # -*- coding: utf-8 -*- # @author : Quan Liu # @date : 2020/5/21 16:01 import time import tensorflow as tf from tqdm import tqdm from seq2seq.model_seq2seq_lq import Seq2Seq from seq2seq.model_utils import get_input_from_batch, get_output_from_batch from utils import config from utils.common_utils import init_logger logger = init_logger() def eval_model(batcher, model: Seq2Seq, ckpt_manager: tf.train.CheckpointManager) -> float: # 训练参数 batch_size = config.eval_batch_size learning_rate = config.lr # 优化器 optimizer = tf.keras.optimizers.Adam(name='Adam', learning_rate=learning_rate) # 训练 # @tf.function def train_step(batcher) -> float: with tf.GradientTape() as tape: