def symbols_to_logits_fn(ids, i, cache): nonlocal unable_cache ids = ids[:, -1:] target = tf.expand_dims(ids, axis=-1) # [batch,1,1] embedding_target, _ = embedding(target, conf.vocab_size, conf.hidden_size, 'share_embedding', reuse=True) input = tf.squeeze(embedding_target, axis=1) # [batch,hid] # 合并 cache和unable_cache为state state = cell_decoder.zero_state(batch_size * conf.beam_size, tf.float32).clone( cell_state=cache['cell_state'], attention=cache['attention'], alignments=cache['alignments'], attention_state=cache['attention_state'], alignment_history=unable_cache['alignment_history'], # time=unable_cache['time'], time=tf.convert_to_tensor(i, dtype=tf.int32), ) with tf.variable_scope('decoder/rnn', reuse=tf.AUTO_REUSE): output, state = cell_decoder(input, state) # 分开cache和unable_cache cache['cell_state'] = state.cell_state cache['attention'] = state.attention cache['alignments'] = state.alignments cache['attention_state'] = state.attention_state unable_cache['alignment_history'] = state.alignment_history # unable_cache['time'] = state.time body_output = output # [batch,hidden] logits = proj_logits(body_output, conf.embed_size, conf.vocab_size, name='share_embedding') return logits, cache
def build_model2(self): # embedding # [batch,len,embed] # pretrained_word_embeddings = np.load(f'{curr_dir}/pretrain_emb_300.npy') pretrained_word_embeddings = None s1_embed, _ = embedding(tf.expand_dims(self.s1, -1), conf.vocab_size, conf.embed_size, name='embedding', pretrain_embedding=pretrained_word_embeddings) s1_mask = mask_nonpad_from_embedding( s1_embed) # [batch,len] 1 for nonpad; 0 for pad s1_seqlen = tf.cast(tf.reduce_sum(s1_mask, axis=-1), tf.int32) # [batch] # encoder encoder_input = s1_embed encoder_input = tf.layers.dropout(encoder_input, self.dropout_rate) # dropout idcnn_net = IDCNN() encoder_output = idcnn_net(encoder_input) encoder_output = tf.layers.dropout(encoder_output, self.dropout_rate) # dropout # logits cls_logits = tf.layers.dense(encoder_output, conf.label_size, activation=None, use_bias=True, name='cls_logits') # [batch,length,label] # crf log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood( cls_logits, self.ner_label, tf.to_int32(s1_seqlen)) crf_loss = tf.reduce_mean(-log_likelihood) # loss self.loss = crf_loss # crf decode ner_pred, ner_prob = tf.contrib.crf.crf_decode(cls_logits, transition_params, tf.to_int32(s1_seqlen)) self.ner_prob = tf.identity(ner_prob, name='ner_prob') # [batch] self.ner_pred = tf.identity(ner_pred, name='ner_pred') # [batch,len] with tf.name_scope('accuracy'): ner_acc = tf.cast(tf.equal(self.ner_pred, self.ner_label), tf.float32) * s1_mask # [batch,len] ner_acc = tf.reduce_sum(ner_acc, axis=-1) # [batch] ner_acc = ner_acc / tf.cast(s1_seqlen, tf.float32) # [batch] self.accuracy = tf.reduce_mean(ner_acc, axis=0) # scalar self.global_step = tf.train.get_or_create_global_step() self.optimizer = tf.train.AdamOptimizer(learning_rate=conf.lr) self.train_op = self.optimizer.minimize(self.loss, global_step=self.global_step)
def symbols_to_logits_fn(ids, i, cache): # ids [batch,length] pred_target = ids[:, -1:] # [batch,1] 截取最后一个 target_embed, _ = embedding(tf.expand_dims(pred_target, axis=-1), conf.vocab_size, conf.embed_size, 'share_embedding') # [batch,1,embed] decoder_input = tf.squeeze(target_embed, axis=1) # [batch,embed] # 输入加上ctx decoder_input = tf.concat([decoder_input, cache['ctx']], axis=-1) # run rnn decoder_output, cache['state'] = self.decoder_rnn.one_step(decoder_input, cache['state']) logits = proj_logits(decoder_output, conf.embed_size, conf.vocab_size, name='share_embedding') return logits, cache
def build_model2(self): # pretrained_word_embeddings = np.load(f'{curr_dir}/pretrain_emb_300.npy') pretrained_word_embeddings = None s1_embed, _ = embedding(tf.expand_dims(self.s1, -1), conf.vocab_size, conf.embed_size, pretrain_embedding=pretrained_word_embeddings) """ encoder """ # encoder_input # [batch,length,embed] encoder_input = s1_embed encoder_valid_mask = mask_nonpad_from_embedding(encoder_input) # [batch,length] 1 for nonpad; 0 for pad encoder_input = add_timing_signal_1d(encoder_input) # add position embedding encoder_input = tf.layers.dropout(encoder_input, self.dropout_rate) # dropout encoder_output = transformer_encoder(encoder_input, encoder_valid_mask, hidden_size=conf.hidden_size, filter_size=conf.hidden_size * 4, num_heads=conf.num_heads, num_encoder_layers=conf.num_encoder_layers, dropout=self.dropout_rate, attention_dropout=self.dropout_rate, relu_dropout=self.dropout_rate, ) # attention # e.g. HAN attn_output = multi_head_attention_base_pool(encoder_output, encoder_valid_mask, conf.embed_size, conf.embed_size, 6, name='sent_pool', reuse=tf.AUTO_REUSE, dropout=self.dropout_rate, attn_v=None) self.logits = tf.layers.dense(attn_output, conf.label_size, use_bias=True, name='logits') # [batch,cls] self.one_hot_target = tf.one_hot(self.target, conf.label_size) smooth_one_hot_target = label_smoothing(self.one_hot_target) self.loss = softmax_cross_entropy(self.logits, smooth_one_hot_target) self.y_prob = tf.nn.softmax(self.logits) self.y_prob = tf.identity(self.y_prob, name='y_prob') with tf.name_scope("accuracy"): self.correct = tf.equal( tf.argmax(self.logits, -1, output_type=tf.int32), self.target) self.accuracy = tf.reduce_mean(tf.cast(self.correct, tf.float32)) self.global_step = tf.train.get_or_create_global_step() self.optimizer = tf.train.AdamOptimizer(learning_rate=conf.lr) self.train_op = self.optimizer.minimize(self.loss, global_step=self.global_step)
def symbols_to_logits_fn(ids, i, cache): # ids [batch,length] pred_target = ids[:, -1:] # 截取最后一个 [batch,1] embed_target, _ = embedding(tf.expand_dims(pred_target, axis=-1), conf.vocab_size, conf.embed_size, 'share_embedding') # [batch,length,embed] decoder_input = tf.squeeze(embed_target, axis=1) # [batch,embed] # if use attention s = cache['state'] if isinstance(decoder_rnn, tf.nn.rnn_cell.GRUCell) else cache['state'].h context = encdec_atten(s, beam_size=conf.beam_size) # [batch,hidden] decoder_input = tf.concat([decoder_input, context], axis=-1) # [batch,hidden+] # run rnn # with tf.variable_scope('rnn', reuse=tf.AUTO_REUSE): decoder_output, cache['state'] = decoder_rnn(decoder_input, cache['state']) logits = proj_logits(decoder_output, conf.hidden_size, conf.vocab_size, name='share_embedding') return logits, cache
def symbols_to_logits_fn(ids, i, cache): ids = ids[:, -1:] # [batch,1] 截取最后一个 target_embed, _ = embedding(tf.expand_dims(ids, axis=-1), conf.vocab_size, conf.embed_size, 'share_embedding', reuse=True) # [batch,1,hidden] decoder_input = target_embed + position_embedding[:, i:i + 1, :] # [batch,1,hidden] encoder_output = cache['encoder_output'] encoder_mask = cache['encoder_mask'] with tf.variable_scope('', reuse=tf.AUTO_REUSE): decoder_output = transformer_decoder(decoder_input, encoder_output, None, encoder_mask, cache=cache, # 注意infer要cache hidden_size=conf.embed_size, filter_size=conf.embed_size * 4, num_heads=6, num_decoder_layers=6, ) logits = proj_logits(decoder_output, conf.embed_size, conf.vocab_size, name='share_embedding') # [batch,1,vocab] ret = tf.squeeze(logits, axis=1) # [batch,vocab] return ret, cache
def build_model1(self): self.uttn_enc_hidden_size = 256 self.ctx_enc_hidden_size = 256 batch_size, num_turns, length = shape_list(self.multi_s1) # embedding # [batch,len,turn,embed] multi_s1_embed, _ = embedding(tf.expand_dims(self.multi_s1, -1), conf.vocab_size, conf.embed_size, name='share_embedding', pretrain_embedding=conf.pretrain_emb) # [batch,len,embed] s2_embed, _ = embedding(tf.expand_dims(self.s2, -1), conf.vocab_size, conf.embed_size, name='share_embedding', pretrain_embedding=conf.pretrain_emb) # uttn encoder uttn_input = tf.reshape(multi_s1_embed, [-1, length, conf.embed_size]) # [batch*turn,len,embed] uttn_mask = mask_nonpad_from_embedding(uttn_input) # [batch*turn,len] 1 for nonpad; 0 for pad uttn_seqlen = tf.cast(tf.reduce_sum(uttn_mask, axis=-1), tf.int32) # [batch*turn] # uttn-gru self.encoder_uttn_rnn = RNN(cell_name='GRUCell', name='uttn_enc', hidden_size=self.uttn_enc_hidden_size, dropout_rate=self.dropout_rate) _, uttn_embed = self.encoder_uttn_rnn(uttn_input, uttn_seqlen) # [batch*turn,hid] uttn_embed = tf.reshape(uttn_embed, [batch_size, num_turns, self.uttn_enc_hidden_size]) # [batch,turn,hid] # 之后turn相当于len # ctx encoder ctx_mask = mask_nonpad_from_embedding(uttn_embed) # [batch,turn] 1 for nonpad; 0 for pad ctx_seqlen = tf.cast(tf.reduce_sum(ctx_mask, axis=-1), tf.int32) # [batch] # ctx-gru self.encoder_ctx_rnn = RNN(cell_name='GRUCell', name='ctx_enc', hidden_size=self.ctx_enc_hidden_size, dropout_rate=self.dropout_rate) _, ctx_embed = self.encoder_ctx_rnn(uttn_embed, ctx_seqlen) # [batch,hid] # rnn decoder train (no attention) s2_mask = mask_nonpad_from_embedding(s2_embed) # [batch,len] 1 for nonpad; 0 for pad s2_seqlen = tf.cast(tf.reduce_sum(s2_mask, axis=-1), tf.int32) # [batch] decoder_input = shift_right(s2_embed) # 用pad当做eos decoder_input = tf.layers.dropout(decoder_input, rate=self.dropout_rate) # dropout # 输入拼上ctx decoder_ctx = tf.tile(tf.expand_dims(ctx_embed, axis=1), [1, shape_list(decoder_input)[1], 1]) # [batch,len,hid] decoder_input = tf.concat([decoder_input, decoder_ctx], axis=2) self.decoder_rnn = RNN(cell_name='GRUCell', name='dec', hidden_size=conf.embed_size, dropout_rate=self.dropout_rate) decoder_output, decoder_state = self.decoder_rnn(decoder_input, s2_seqlen) logits = proj_logits(decoder_output, conf.embed_size, conf.vocab_size, name='share_embedding') onehot_s2 = tf.one_hot(self.s2, depth=conf.vocab_size) # [batch,len,vocab] xentropy = tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=onehot_s2) # [batch,len] weights = tf.to_float(tf.not_equal(self.s2, 0)) # [batch,len] 1 for nonpad; 0 for pad loss_num = xentropy * weights # [batch,len] loss_den = weights # [batch,len] loss = tf.reduce_sum(loss_num) / tf.reduce_sum(loss_den) # scalar self.loss = loss # rnn decoder infer (no attention) # 放在cache里面的在后面symbols_to_logits_fn函数中都会变成batch * beam cache = {'state': self.decoder_rnn.cell.zero_state(batch_size, tf.float32), # [batch,hid] 'ctx': ctx_embed, # [batch,hid] } def symbols_to_logits_fn(ids, i, cache): # ids [batch,length] pred_target = ids[:, -1:] # [batch,1] 截取最后一个 target_embed, _ = embedding(tf.expand_dims(pred_target, axis=-1), conf.vocab_size, conf.embed_size, 'share_embedding') # [batch,1,embed] decoder_input = tf.squeeze(target_embed, axis=1) # [batch,embed] # 输入加上ctx decoder_input = tf.concat([decoder_input, cache['ctx']], axis=-1) # run rnn decoder_output, cache['state'] = self.decoder_rnn.one_step(decoder_input, cache['state']) logits = proj_logits(decoder_output, conf.embed_size, conf.vocab_size, name='share_embedding') return logits, cache initial_ids = tf.zeros([batch_size], dtype=tf.int32) # <pad>为<sos> def greedy_search_wrapper(): """ Greedy Search """ decoded_ids, scores = greedy_search( symbols_to_logits_fn, initial_ids, conf.max_decode_len, cache=cache, eos_id=conf.eos_id, ) return decoded_ids, scores def beam_search_wrapper(): """ Beam Search """ decoded_ids, scores = beam_search( # [batch,beam,len] [batch,beam] symbols_to_logits_fn, initial_ids, conf.beam_size, conf.max_decode_len, conf.vocab_size, alpha=0, states=cache, eos_id=conf.eos_id, ) return decoded_ids, scores decoded_ids, scores = tf.cond(tf.equal(conf.beam_size, 1), greedy_search_wrapper, beam_search_wrapper) self.decoded_ids = tf.identity(decoded_ids, name='decoded_ids') # [batch,beam/1,len] self.scores = tf.identity(scores, name='scores') # [batch,beam/1] self.global_step = tf.train.get_or_create_global_step() self.optimizer = tf.train.AdamOptimizer(learning_rate=conf.lr) self.train_op = self.optimizer.minimize(self.loss, global_step=self.global_step)
def build_model3(self): assert not conf.hidden_size % 2 # 被2整除 self.uttn_enc_hidden_size = conf.hidden_size // 2 batch_size, num_turns, length = shape_list(self.multi_s1) # embedding # [batch,len,turn,embed] multi_s1_embed, _ = embedding(tf.expand_dims(self.multi_s1, -1), conf.vocab_size, conf.embed_size, name='share_embedding', pretrain_embedding=conf.pretrain_emb) # [batch,len,embed] s2_embed, _ = embedding(tf.expand_dims(self.s2, -1), conf.vocab_size, conf.embed_size, name='share_embedding', pretrain_embedding=conf.pretrain_emb) # uttn encoder uttn_input = tf.reshape(multi_s1_embed, [-1, length, conf.embed_size]) # [batch*turn,len,embed] uttn_mask = mask_nonpad_from_embedding(uttn_input) # [batch*turn,len] 1 for nonpad; 0 for pad uttn_seqlen = tf.cast(tf.reduce_sum(uttn_mask, axis=-1), tf.int32) # [batch*turn] # uttn-gru self.encoder_uttn_rnn = Bi_RNN(cell_name='GRUCell', name='uttn_enc', hidden_size=self.uttn_enc_hidden_size, dropout_rate=self.dropout_rate) _, uttn_embed = self.encoder_uttn_rnn(uttn_input, uttn_seqlen) # [batch*turn,len,2hid] [batch*turn,2hid] uttn_embed = tf.reshape(uttn_embed, [batch_size, num_turns, self.uttn_enc_hidden_size * 2]) # [batch,turn,2hid] # 之后turn相当于len # transformer ctx encoder encoder_valid_mask = mask_nonpad_from_embedding(uttn_embed) # [batch,turn] 1 for nonpad; 0 for pad encoder_input = add_timing_signal_1d(uttn_embed) # add position embedding encoder_input = tf.layers.dropout(encoder_input, rate=self.dropout_rate) # dropout encoder_output = transformer_encoder(encoder_input, encoder_valid_mask, hidden_size=conf.hidden_size, filter_size=conf.hidden_size * 4, num_heads=conf.num_heads, num_encoder_layers=conf.num_encoder_layers, dropout=self.dropout_rate, attention_dropout=self.dropout_rate, relu_dropout=self.dropout_rate, ) # transformer decoder decoder_input = s2_embed decoder_valid_mask = mask_nonpad_from_embedding(decoder_input) # [batch,len] 1 for nonpad; 0 for pad decoder_input = shift_right(decoder_input) # 用pad当做eos decoder_input = add_timing_signal_1d(decoder_input) decoder_input = tf.layers.dropout(decoder_input, rate=self.dropout_rate) # dropout decoder_output = transformer_decoder(decoder_input, encoder_output, decoder_valid_mask, encoder_valid_mask, cache=None, hidden_size=conf.hidden_size, filter_size=conf.hidden_size * 4, num_heads=conf.num_heads, num_decoder_layers=conf.num_decoder_layers, dropout=self.dropout_rate, attention_dropout=self.dropout_rate, relu_dropout=self.dropout_rate, ) logits = proj_logits(decoder_output, conf.embed_size, conf.vocab_size, name='share_embedding') onehot_s2 = tf.one_hot(self.s2, depth=conf.vocab_size) # [batch,len,vocab] xentropy = tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=onehot_s2) # [batch,len] weights = tf.to_float(tf.not_equal(self.s2, 0)) # [batch,len] 1 for nonpad; 0 for pad loss_num = xentropy * weights # [batch,len] loss_den = weights # [batch,len] loss = tf.reduce_sum(loss_num) / tf.reduce_sum(loss_den) # scalar self.loss = loss # transformer decoder infer # 放在cache里面的在后面symbols_to_logits_fn函数中都会变成batch * beam # 初始化缓存 cache = { 'layer_%d' % layer: { # 用以缓存decoder过程前面已计算的k,v 'k': split_heads(tf.zeros([batch_size, 0, conf.embed_size]), conf.num_heads), 'v': split_heads(tf.zeros([batch_size, 0, conf.embed_size]), conf.num_heads) } for layer in range(conf.num_decoder_layers) } for layer in range(conf.num_decoder_layers): # 对于decoder每层均需与encoder顶层隐状态计算attention,相应均有特定的k,v可缓存 layer_name = 'layer_%d' % layer with tf.variable_scope('decoder/%s/encdec_attention/multihead_attention' % layer_name): k_encdec = tf.layers.dense(encoder_output, conf.embed_size, use_bias=False, name='k', reuse=tf.AUTO_REUSE) k_encdec = split_heads(k_encdec, conf.num_heads) v_encdec = tf.layers.dense(encoder_output, conf.embed_size, use_bias=False, name='v', reuse=tf.AUTO_REUSE) v_encdec = split_heads(v_encdec, conf.num_heads) cache[layer_name]['k_encdec'] = k_encdec cache[layer_name]['v_encdec'] = v_encdec cache['encoder_output'] = encoder_output cache['encoder_mask'] = encoder_valid_mask # position embedding position_embedding = get_timing_signal_1d(conf.max_decode_len, conf.embed_size) # +eos [1,length+1,embed] def symbols_to_logits_fn(ids, i, cache): ids = ids[:, -1:] # [batch,1] 截取最后一个 target_embed, _ = embedding(tf.expand_dims(ids, axis=-1), conf.vocab_size, conf.embed_size, 'share_embedding', reuse=True) # [batch,1,hidden] decoder_input = target_embed + position_embedding[:, i:i + 1, :] # [batch,1,hidden] encoder_output = cache['encoder_output'] encoder_mask = cache['encoder_mask'] with tf.variable_scope('', reuse=tf.AUTO_REUSE): decoder_output = transformer_decoder(decoder_input, encoder_output, None, encoder_mask, cache=cache, # 注意infer要cache hidden_size=conf.embed_size, filter_size=conf.embed_size * 4, num_heads=6, num_decoder_layers=6, ) logits = proj_logits(decoder_output, conf.embed_size, conf.vocab_size, name='share_embedding') # [batch,1,vocab] ret = tf.squeeze(logits, axis=1) # [batch,vocab] return ret, cache initial_ids = tf.zeros([batch_size], dtype=tf.int32) # <pad>为<sos> def greedy_search_wrapper(): """ Greedy Search """ decoded_ids, scores = greedy_search( symbols_to_logits_fn, initial_ids, conf.max_decode_len, cache=cache, eos_id=conf.eos_id, ) return decoded_ids, scores def beam_search_wrapper(): """ Beam Search """ decoded_ids, scores = beam_search( # [batch,beam,len] [batch,beam] symbols_to_logits_fn, initial_ids, conf.beam_size, conf.max_decode_len, conf.vocab_size, alpha=0, states=cache, eos_id=conf.eos_id, ) return decoded_ids, scores decoded_ids, scores = tf.cond(tf.equal(conf.beam_size, 1), greedy_search_wrapper, beam_search_wrapper) self.decoded_ids = tf.identity(decoded_ids, name='decoded_ids') # [batch,beam/1,len] self.scores = tf.identity(scores, name='scores') # [batch,beam/1] self.global_step = tf.train.get_or_create_global_step() self.optimizer = tf.train.AdamOptimizer(learning_rate=conf.lr) self.train_op = self.optimizer.minimize(self.loss, global_step=self.global_step)
def symbols_to_logits_fn(ids, i, cache): # ids [batch,length] pred_target = ids[:, -1:] # [batch,1] 截取最后一个 target_embed, _ = embedding(tf.expand_dims(pred_target, axis=-1), conf.vocab_size, conf.embed_size, 'share_embedding') # [batch,1,embed] decoder_input = tf.squeeze(target_embed, axis=1) # [batch,embed] dec_rnn_state = cache['dec_rnn_state'] with tf.variable_scope('', reuse=tf.AUTO_REUSE): # 内层循环是对于解码每一步 ctx_rnn上turn的每一步,最终生成ctx序列 turn_step def inner_loop_condition(turn_step, *_): return tf.less(turn_step, tf.reduce_max(ctx_seqlen)) def inner_loop_body(turn_step, ctx_rnn_state, ctx_rnn_output): # 根据si, ctx_init_state 递归计算多个turn的句子ctx q_antecedent = tf.concat([ctx_rnn_state, dec_rnn_state], axis=-1) # [batch, h] q_antecedent = tf.tile(tf.expand_dims(q_antecedent, 1), [1, length, 1]) # [batch,len,h] # 抽取每个batch的第i个turn # sent_repre [batch,turn,len,h] q_antecedent = tf.concat([cache['uttn_repre'][:, turn_step, :, :], q_antecedent], -1) # [batch,len,h] uttn_mask_in_turn = tf.reshape(uttn_mask, [batch_size, num_turns, length])[:, turn_step, :] # [batch,len] # word-level-attn h = tf.layers.dense(q_antecedent, 128, activation=tf.nn.tanh, use_bias=True, name='word_level_attn/layer1') energy = tf.layers.dense(h, 1, use_bias=True, name='word_level_attn/layer2') # [batch,len,1] energy = tf.squeeze(energy, -1) + (1. - uttn_mask_in_turn) * -1e9 alpha = tf.nn.softmax(energy) # [batch,len] r_in_turn = tf.reduce_sum(tf.expand_dims(alpha, -1) * cache['uttn_repre'][:, turn_step, :, :], 1) # [batch,h] ctx_rnn_output_, ctx_rnn_state = self.encoder_ctx_rnn.one_step(r_in_turn, ctx_rnn_state) # attch ctx_rnn_output = tf.concat([ctx_rnn_output, tf.expand_dims(ctx_rnn_output_, 1)], 1) # [batch,turn,h] return turn_step + 1, ctx_rnn_state, ctx_rnn_output # start inner loop final_turn_step, final_state, ctx_rnn_output = tf.while_loop( inner_loop_condition, inner_loop_body, loop_vars=[tf.constant(0, dtype=tf.int32), cache['ctx_rnn_state'], tf.zeros([shape_list(cache['ctx_rnn_state'])[0], 0, self.ctx_enc_hidden_size])], shape_invariants=[ tf.TensorShape([]), nest.map_structure(get_state_shape_invariants, init_ctx_encoder_state), tf.TensorShape([None, None, self.ctx_enc_hidden_size]), ]) # ctx_rnn_output # [batch,turn,h] # dec_rnn_state # [batch,h] # ctx-level-attn # q_antecedent = tf.tile(tf.expand_dims(dec_rnn_state, axis=1), [1, num_turns, 1]) # [batch,turn,h] # 这样只拿当前batch中的尽可能小的turns数量而不是固定turn q_antecedent = tf.tile(tf.expand_dims(dec_rnn_state, axis=1), [1, shape_list(ctx_rnn_output)[1], 1]) # [batch,turn,h] q_antecedent = tf.concat([q_antecedent, ctx_rnn_output], 2) # [batch,turn,h] h = tf.layers.dense(q_antecedent, 128, activation=tf.nn.tanh, use_bias=True, name='ctx_level_attn/layer1') energy = tf.layers.dense(h, 1, use_bias=True, name='ctx_level_attn/layer2') # [batch,turn,1] energy = tf.squeeze(energy, -1) + (1. - ctx_mask) * -1e9 # [batch,turn] alpha = tf.nn.softmax(energy) # [batch,turn] ctx_input_in_dec = tf.reduce_sum(tf.expand_dims(alpha, -1) * ctx_rnn_output, 1) # [batch,h] dec_rnn_input = tf.concat([ctx_input_in_dec, decoder_input], -1) # [batch,h] dec_rnn_output_, dec_rnn_state = self.decoder_rnn.one_step(dec_rnn_input, dec_rnn_state) cache['dec_rnn_state'] = dec_rnn_state logits = proj_logits(dec_rnn_output_, conf.embed_size, conf.vocab_size, name='share_embedding') return logits, cache
def build_model2(self): self.uttn_enc_hidden_size = 256 self.ctx_enc_hidden_size = 256 batch_size, num_turns, length = shape_list(self.multi_s1) # embedding # [batch,len,turn,embed] multi_s1_embed, _ = embedding(tf.expand_dims(self.multi_s1, -1), conf.vocab_size, conf.embed_size, name='share_embedding', pretrain_embedding=conf.pretrain_emb) # [batch,len,embed] s2_embed, _ = embedding(tf.expand_dims(self.s2, -1), conf.vocab_size, conf.embed_size, name='share_embedding', pretrain_embedding=conf.pretrain_emb) # uttn encoder uttn_input = tf.reshape(multi_s1_embed, [-1, length, conf.embed_size]) # [batch*turn,len,embed] uttn_mask = mask_nonpad_from_embedding(uttn_input) # [batch*turn,len] 1 for nonpad; 0 for pad uttn_seqlen = tf.cast(tf.reduce_sum(uttn_mask, axis=-1), tf.int32) # [batch*turn] # uttn-gru self.encoder_uttn_rnn = Bi_RNN(cell_name='GRUCell', name='uttn_enc', hidden_size=self.uttn_enc_hidden_size, dropout_rate=self.dropout_rate) uttn_repre, uttn_embed = self.encoder_uttn_rnn(uttn_input, uttn_seqlen) # [batch*turn,len,2hid] [batch*turn,2hid] uttn_embed = tf.reshape(uttn_embed, [batch_size, num_turns, self.uttn_enc_hidden_size * 2]) # [batch,turn,2hid] uttn_repre = tf.reshape(uttn_repre, [batch_size, num_turns, length, self.uttn_enc_hidden_size * 2]) # [batch,turn,len,2hid] # ctx encoder ctx_mask = mask_nonpad_from_embedding(uttn_embed) # [batch,turn] 1 for nonpad; 0 for pad ctx_seqlen = tf.cast(tf.reduce_sum(ctx_mask, axis=-1), tf.int32) # [batch] # reverse turn uttn_repre = tf.reverse_sequence(uttn_repre, seq_lengths=ctx_seqlen, seq_axis=1, batch_axis=0) # ctx-gru self.encoder_ctx_rnn = RNN(cell_name='GRUCell', name='ctx_enc', hidden_size=self.ctx_enc_hidden_size, dropout_rate=self.dropout_rate) init_ctx_encoder_state = self.encoder_ctx_rnn.cell.zero_state(batch_size, tf.float32) # rnn decoder train s2_mask = mask_nonpad_from_embedding(s2_embed) # [batch,len] 1 for nonpad; 0 for pad s2_seqlen = tf.cast(tf.reduce_sum(s2_mask, axis=-1), tf.int32) # [batch] decoder_input = shift_right(s2_embed) # 用pad当做eos decoder_input = tf.layers.dropout(decoder_input, rate=self.dropout_rate) # dropout self.decoder_rnn = RNN(cell_name='GRUCell', name='dec', hidden_size=conf.embed_size, dropout_rate=self.dropout_rate) init_decoder_state = self.decoder_rnn.cell.zero_state(batch_size, tf.float32) # 两重循环 -_- # 外层循环是decoder解码的每一步 time_step def loop_condition(time_step, *_): return tf.less(time_step, tf.reduce_max(s2_seqlen)) def loop_body(time_step, dec_rnn_state, dec_rnn_output): # cal decoder ctx # word level attention # 内层循环是对于解码每一步 ctx_rnn上turn的每一步,最终生成ctx序列 turn_step def inner_loop_condition(turn_step, *_): return tf.less(turn_step, tf.reduce_max(ctx_seqlen)) def inner_loop_body(turn_step, ctx_rnn_state, ctx_rnn_output): # 根据si, ctx_init_state 递归计算多个turn的句子ctx q_antecedent = tf.concat([ctx_rnn_state, dec_rnn_state], axis=-1) # [batch, h] q_antecedent = tf.tile(tf.expand_dims(q_antecedent, 1), [1, length, 1]) # [batch,len,h] # 抽取每个batch的第i个turn # sent_repre [batch,turn,len,h] q_antecedent = tf.concat([uttn_repre[:, turn_step, :, :], q_antecedent], -1) # [batch,len,h] uttn_mask_in_turn = tf.reshape(uttn_mask, [batch_size, num_turns, length])[:, turn_step, :] # [batch,len] # word-level-attn h = tf.layers.dense(q_antecedent, 128, activation=tf.nn.tanh, use_bias=True, name='word_level_attn/layer1') energy = tf.layers.dense(h, 1, use_bias=True, name='word_level_attn/layer2') # [batch,len,1] energy = tf.squeeze(energy, -1) + (1. - uttn_mask_in_turn) * -1e9 alpha = tf.nn.softmax(energy) # [batch,len] r_in_turn = tf.reduce_sum(tf.expand_dims(alpha, -1) * uttn_repre[:, turn_step, :, :], 1) # [batch,h] ctx_rnn_output_, ctx_rnn_state = self.encoder_ctx_rnn.one_step(r_in_turn, ctx_rnn_state) # attch ctx_rnn_output = tf.concat([ctx_rnn_output, tf.expand_dims(ctx_rnn_output_, 1)], 1) # [batch,turn,h] return turn_step + 1, ctx_rnn_state, ctx_rnn_output # start inner loop final_turn_step, final_state, ctx_rnn_output = tf.while_loop( inner_loop_condition, inner_loop_body, loop_vars=[tf.constant(0, dtype=tf.int32), init_ctx_encoder_state, tf.zeros([batch_size, 0, self.ctx_enc_hidden_size])], shape_invariants=[ tf.TensorShape([]), nest.map_structure(get_state_shape_invariants, init_ctx_encoder_state), tf.TensorShape([None, None, self.ctx_enc_hidden_size]), ]) # ctx_rnn_output # [batch,turn,h] # dec_rnn_state # [batch,h] # ctx-level-attn # q_antecedent = tf.tile(tf.expand_dims(dec_rnn_state, axis=1), [1, num_turns, 1]) # [batch,turn,h] # 这样只拿当前batch中的尽可能小的turns数量而不是固定turn q_antecedent = tf.tile(tf.expand_dims(dec_rnn_state, axis=1), [1, shape_list(ctx_rnn_output)[1], 1]) # [batch,turn,h] q_antecedent = tf.concat([q_antecedent, ctx_rnn_output], 2) # [batch,turn,h] h = tf.layers.dense(q_antecedent, 128, activation=tf.nn.tanh, use_bias=True, name='ctx_level_attn/layer1') energy = tf.layers.dense(h, 1, use_bias=True, name='ctx_level_attn/layer2') # [batch,turn,1] energy = tf.squeeze(energy, -1) + (1. - ctx_mask) * -1e9 # [batch,turn] alpha = tf.nn.softmax(energy) # [batch,turn] ctx_input_in_dec = tf.reduce_sum(tf.expand_dims(alpha, -1) * ctx_rnn_output, 1) # [batch,h] dec_rnn_input = tf.concat([ctx_input_in_dec, decoder_input[:, time_step, :]], -1) # [batch,h] dec_rnn_output_, dec_rnn_state = self.decoder_rnn.one_step(dec_rnn_input, dec_rnn_state) dec_rnn_output = tf.concat([dec_rnn_output, tf.expand_dims(dec_rnn_output_, 1)], 1) return time_step + 1, dec_rnn_state, dec_rnn_output # start outer loop final_time_step, final_state, dec_rnn_output = tf.while_loop( loop_condition, loop_body, loop_vars=[tf.constant(0, dtype=tf.int32), init_decoder_state, tf.zeros([batch_size, 0, conf.embed_size])], shape_invariants=[ tf.TensorShape([]), nest.map_structure(get_state_shape_invariants, init_decoder_state), tf.TensorShape([None, None, conf.embed_size]), ]) decoder_output = dec_rnn_output logits = proj_logits(decoder_output, conf.embed_size, conf.vocab_size, name='share_embedding') onehot_s2 = tf.one_hot(self.s2, depth=conf.vocab_size) # [batch,len,vocab] xentropy = tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=onehot_s2) # [batch,len] weights = tf.to_float(tf.not_equal(self.s2, 0)) # [batch,len] 1 for nonpad; 0 for pad loss_num = xentropy * weights # [batch,len] loss_den = weights # [batch,len] loss = tf.reduce_sum(loss_num) / tf.reduce_sum(loss_den) # scalar self.loss = loss # rnn decoder infer # 放在cache里面的在后面symbols_to_logits_fn函数中都会变成batch * beam cache = {'dec_rnn_state': self.decoder_rnn.cell.zero_state(batch_size, tf.float32), # [batch,hid] 'ctx_rnn_state': self.encoder_ctx_rnn.cell.zero_state(batch_size, tf.float32), # [batch,hid] 'uttn_repre': uttn_repre # [batch,turn,len,2hid] } def symbols_to_logits_fn(ids, i, cache): # ids [batch,length] pred_target = ids[:, -1:] # [batch,1] 截取最后一个 target_embed, _ = embedding(tf.expand_dims(pred_target, axis=-1), conf.vocab_size, conf.embed_size, 'share_embedding') # [batch,1,embed] decoder_input = tf.squeeze(target_embed, axis=1) # [batch,embed] dec_rnn_state = cache['dec_rnn_state'] with tf.variable_scope('', reuse=tf.AUTO_REUSE): # 内层循环是对于解码每一步 ctx_rnn上turn的每一步,最终生成ctx序列 turn_step def inner_loop_condition(turn_step, *_): return tf.less(turn_step, tf.reduce_max(ctx_seqlen)) def inner_loop_body(turn_step, ctx_rnn_state, ctx_rnn_output): # 根据si, ctx_init_state 递归计算多个turn的句子ctx q_antecedent = tf.concat([ctx_rnn_state, dec_rnn_state], axis=-1) # [batch, h] q_antecedent = tf.tile(tf.expand_dims(q_antecedent, 1), [1, length, 1]) # [batch,len,h] # 抽取每个batch的第i个turn # sent_repre [batch,turn,len,h] q_antecedent = tf.concat([cache['uttn_repre'][:, turn_step, :, :], q_antecedent], -1) # [batch,len,h] uttn_mask_in_turn = tf.reshape(uttn_mask, [batch_size, num_turns, length])[:, turn_step, :] # [batch,len] # word-level-attn h = tf.layers.dense(q_antecedent, 128, activation=tf.nn.tanh, use_bias=True, name='word_level_attn/layer1') energy = tf.layers.dense(h, 1, use_bias=True, name='word_level_attn/layer2') # [batch,len,1] energy = tf.squeeze(energy, -1) + (1. - uttn_mask_in_turn) * -1e9 alpha = tf.nn.softmax(energy) # [batch,len] r_in_turn = tf.reduce_sum(tf.expand_dims(alpha, -1) * cache['uttn_repre'][:, turn_step, :, :], 1) # [batch,h] ctx_rnn_output_, ctx_rnn_state = self.encoder_ctx_rnn.one_step(r_in_turn, ctx_rnn_state) # attch ctx_rnn_output = tf.concat([ctx_rnn_output, tf.expand_dims(ctx_rnn_output_, 1)], 1) # [batch,turn,h] return turn_step + 1, ctx_rnn_state, ctx_rnn_output # start inner loop final_turn_step, final_state, ctx_rnn_output = tf.while_loop( inner_loop_condition, inner_loop_body, loop_vars=[tf.constant(0, dtype=tf.int32), cache['ctx_rnn_state'], tf.zeros([shape_list(cache['ctx_rnn_state'])[0], 0, self.ctx_enc_hidden_size])], shape_invariants=[ tf.TensorShape([]), nest.map_structure(get_state_shape_invariants, init_ctx_encoder_state), tf.TensorShape([None, None, self.ctx_enc_hidden_size]), ]) # ctx_rnn_output # [batch,turn,h] # dec_rnn_state # [batch,h] # ctx-level-attn # q_antecedent = tf.tile(tf.expand_dims(dec_rnn_state, axis=1), [1, num_turns, 1]) # [batch,turn,h] # 这样只拿当前batch中的尽可能小的turns数量而不是固定turn q_antecedent = tf.tile(tf.expand_dims(dec_rnn_state, axis=1), [1, shape_list(ctx_rnn_output)[1], 1]) # [batch,turn,h] q_antecedent = tf.concat([q_antecedent, ctx_rnn_output], 2) # [batch,turn,h] h = tf.layers.dense(q_antecedent, 128, activation=tf.nn.tanh, use_bias=True, name='ctx_level_attn/layer1') energy = tf.layers.dense(h, 1, use_bias=True, name='ctx_level_attn/layer2') # [batch,turn,1] energy = tf.squeeze(energy, -1) + (1. - ctx_mask) * -1e9 # [batch,turn] alpha = tf.nn.softmax(energy) # [batch,turn] ctx_input_in_dec = tf.reduce_sum(tf.expand_dims(alpha, -1) * ctx_rnn_output, 1) # [batch,h] dec_rnn_input = tf.concat([ctx_input_in_dec, decoder_input], -1) # [batch,h] dec_rnn_output_, dec_rnn_state = self.decoder_rnn.one_step(dec_rnn_input, dec_rnn_state) cache['dec_rnn_state'] = dec_rnn_state logits = proj_logits(dec_rnn_output_, conf.embed_size, conf.vocab_size, name='share_embedding') return logits, cache initial_ids = tf.zeros([batch_size], dtype=tf.int32) # <pad>为<sos> def greedy_search_wrapper(): """ Greedy Search """ decoded_ids, scores = greedy_search( symbols_to_logits_fn, initial_ids, conf.max_decode_len, cache=cache, eos_id=conf.eos_id, ) return decoded_ids, scores def beam_search_wrapper(): """ Beam Search """ decoded_ids, scores = beam_search( # [batch,beam,len] [batch,beam] symbols_to_logits_fn, initial_ids, conf.beam_size, conf.max_decode_len, conf.vocab_size, alpha=0, states=cache, eos_id=conf.eos_id, ) return decoded_ids, scores decoded_ids, scores = tf.cond(tf.equal(conf.beam_size, 1), greedy_search_wrapper, beam_search_wrapper) self.decoded_ids = tf.identity(decoded_ids, name='decoded_ids') # [batch,beam/1,len] self.scores = tf.identity(scores, name='scores') # [batch,beam/1] self.global_step = tf.train.get_or_create_global_step() self.optimizer = tf.train.AdamOptimizer(learning_rate=conf.lr) self.train_op = self.optimizer.minimize(self.loss, global_step=self.global_step)
def build_model1(self): # embedding # [batch,len,embed] s1_embed, _ = embedding(tf.expand_dims(self.s1, -1), conf.vocab_size, conf.embed_size, name='share_embedding', pretrain_embedding=conf.pretrain_emb) s2_embed, _ = embedding(tf.expand_dims(self.s2, -1), conf.vocab_size, conf.embed_size, name='share_embedding', pretrain_embedding=conf.pretrain_emb) s1_input_mask = mask_nonpad_from_embedding(s1_embed) # [batch,len1] 1 for nonpad; 0 for pad s2_input_mask = mask_nonpad_from_embedding(s2_embed) # [batch,len2] 1 for nonpad; 0 for pad s1_seq_len = tf.cast(tf.reduce_sum(s1_input_mask, axis=-1), tf.int32) # [batch] s2_seq_len = tf.cast(tf.reduce_sum(s2_input_mask, axis=-1), tf.int32) # [batch] # bilstm sent encoder self.bilstm_encoder1 = Bi_RNN(cell_name='LSTMCell', hidden_size=conf.birnn_hidden_size, dropout_rate=self.dropout_rate) s1_bar, _ = self.bilstm_encoder1(s1_embed, s1_seq_len) # [batch,len1,2hid] s2_bar, _ = self.bilstm_encoder1(s2_embed, s2_seq_len) # [batch,len2,2hid] # local inference 局部推理 with tf.variable_scope('local_inference'): # 点积注意力 attention_logits = tf.matmul(s1_bar, tf.transpose(s2_bar, [0, 2, 1])) # [batch,len1,len2] # 注意需attention mask pad_mask * -inf + logits attention_s1 = tf.nn.softmax(attention_logits + tf.expand_dims((1. - s2_input_mask) * -1e9, 1)) # [batch,len1,len2] attention_s2 = tf.nn.softmax(tf.transpose(attention_logits, [0, 2, 1]) + tf.expand_dims((1. - s1_input_mask) * -1e9, 1)) # [batch,len2,len1] s1_hat = tf.matmul(attention_s1, s2_bar) # [batch,len1,2hid] s2_hat = tf.matmul(attention_s2, s1_bar) # [batch,len2,2hid] s1_diff = s1_bar - s1_hat s1_mul = s1_bar * s1_hat s2_diff = s2_bar - s2_hat s2_mul = s2_bar * s2_hat m_s1 = tf.concat([s1_bar, s1_hat, s1_diff, s1_mul], axis=2) # [batch,len1,8hid] m_s2 = tf.concat([s2_bar, s2_hat, s2_diff, s2_mul], axis=2) # [batch,len2,8hid] # composition 推理组成 with tf.variable_scope('composition'): self.bilstm_encoder2 = Bi_RNN(cell_name='LSTMCell', hidden_size=conf.birnn_hidden_size, dropout_rate=self.dropout_rate) v_s1, _ = self.bilstm_encoder2(m_s1, s1_seq_len) # [batch,len1,2hid] v_s2, _ = self.bilstm_encoder2(m_s2, s2_seq_len) # [batch,len2,2hid] # average pooling # 需将pad的vector变为0 v_s1 = v_s1 * tf.expand_dims(s1_input_mask, -1) # [batch,len1,2hid] v_s2 = v_s2 * tf.expand_dims(s2_input_mask, -1) # [batch,len1,2hid] v_s1_avg = tf.reduce_sum(v_s1, axis=1) / tf.cast(tf.expand_dims(s1_seq_len, -1), tf.float32) # [batch,2hid] v_s2_avg = tf.reduce_sum(v_s2, axis=1) / tf.cast(tf.expand_dims(s2_seq_len, -1), tf.float32) # [batch,2hid] # max pooling # 需将pad的vector变为极小值 v_s1_max = tf.reduce_max(v_s1 + tf.expand_dims((1. - s1_input_mask) * -1e9, -1), axis=1) # [batch,2hid] v_s2_max = tf.reduce_max(v_s2 + tf.expand_dims((1. - s2_input_mask) * -1e9, -1), axis=1) # [batch,2hid] v = tf.concat([v_s1_avg, v_s1_max, v_s2_avg, v_s2_max], axis=-1) # [batch,8hid] with tf.variable_scope('ffn'): h_ = tf.layers.dropout(v, rate=self.dropout_rate) h = tf.layers.dense(h_, 256, activation=tf.nn.relu, kernel_initializer=tf.random_normal_initializer(0.0, 0.1)) # [batch,256] o_ = tf.layers.dropout(h, rate=self.dropout_rate) o = tf.layers.dense(o_, 1, kernel_initializer=tf.random_normal_initializer(0.0, 0.1)) # [batch,1] self.logits = tf.squeeze(o, -1) # [batch] # loss with tf.name_scope('loss'): loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=tf.cast(self.target, tf.float32), logits=self.logits) # [batch] loss = tf.reduce_mean(loss, -1) # scalar l2_reg = conf.l2_reg weights = [v for v in tf.trainable_variables() if ('w' in v.name) or ('kernel' in v.name)] l2_loss = tf.add_n([tf.nn.l2_loss(w) for w in weights]) * l2_reg loss += l2_loss self.loss = loss self.y_prob = tf.nn.sigmoid(self.logits) self.y_prob = tf.identity(self.y_prob, name='y_prob') with tf.name_scope("accuracy"): self.correct = tf.equal( tf.cast(tf.greater_equal(self.y_prob, 0.5), tf.int32), self.target) self.accuracy = tf.reduce_mean(tf.cast(self.correct, tf.float32)) self.global_step = tf.train.get_or_create_global_step() self.optimizer = tf.train.AdamOptimizer(learning_rate=conf.lr) self.train_op = self.optimizer.minimize(self.loss, global_step=self.global_step)
def build_model3(self): # biGRU encoder + bah_attn + GRU decoder # embedding # [batch,len,embed] # pretrained_word_embeddings = np.load(f'{curr_dir}/pretrain_emb_300.npy') pretrained_word_embeddings = None s1_embed, _ = embedding(tf.expand_dims(self.s1, -1), conf.vocab_size, conf.embed_size, name='share_embedding', pretrain_embedding=pretrained_word_embeddings) s1_mask = mask_nonpad_from_embedding(s1_embed) # [batch,len] 1 for nonpad; 0 for pad s1_seqlen = tf.cast(tf.reduce_sum(s1_mask, axis=-1), tf.int32) # [batch] # encoder encoder_input = s1_embed encoder_input = tf.layers.dropout(encoder_input, rate=self.dropout_rate) # dropout with tf.variable_scope('birnn_encoder'): self.bilstm_encoder1 = Bi_RNN(cell_name='GRUCell', hidden_size=conf.hidden_size, dropout_rate=self.dropout_rate) encoder_output, _ = self.bilstm_encoder1(encoder_input, s1_seqlen) # [batch,len,2hid] batch_size = shape_list(encoder_input)[0] # decoder decoder_rnn = getattr(tf.nn.rnn_cell, 'GRUCell')(conf.hidden_size) # GRUCell/LSTMCell encdec_atten = EncDecAttention(encoder_output, s1_seqlen, conf.hidden_size) s2_embed, _ = embedding(tf.expand_dims(self.s2, -1), conf.vocab_size, conf.embed_size, name='share_embedding', pretrain_embedding=pretrained_word_embeddings) s2_mask = mask_nonpad_from_embedding(s2_embed) # [batch,len] 1 for nonpad; 0 for pad s2_seqlen = tf.cast(tf.reduce_sum(s2_mask, -1), tf.int32) # [batch] decoder_input = s2_embed decoder_input = shift_right(decoder_input) # 用pad当做eos decoder_input = tf.layers.dropout(decoder_input, rate=self.dropout_rate) # dropout init_decoder_state = decoder_rnn.zero_state(batch_size, tf.float32) # init_decoder_state = tf.nn.rnn_cell.LSTMStateTuple(c, h) time_step = tf.constant(0, dtype=tf.int32) rnn_output = tf.zeros([batch_size, 0, conf.hidden_size]) context_output = tf.zeros([batch_size, 0, conf.hidden_size * 2]) # 注意力 def loop_condition(time_step, *_): return tf.less(time_step, tf.reduce_max(s2_seqlen)) def loop_body(time_step, prev_rnn_state, rnn_output, context_output): # attention s = prev_rnn_state if isinstance(decoder_rnn, tf.nn.rnn_cell.GRUCell) else prev_rnn_state.h context = encdec_atten(s) # [batch,hidden] context_output = tf.concat([context_output, tf.expand_dims(context, axis=1)], axis=1) # construct rnn input rnn_input = tf.concat([decoder_input[:, time_step, :], context], axis=-1) # [batch,hidden+] use attention # rnn_input = decoder_input[:, time_step, :] # [batch,hidden] not use attention # run rnn current_output, rnn_state = decoder_rnn(rnn_input, prev_rnn_state) # append to output bucket via length dim rnn_output = tf.concat([rnn_output, tf.expand_dims(current_output, axis=1)], axis=1) return time_step + 1, rnn_state, rnn_output, context_output # start loop final_time_step, final_state, rnn_output, context_output = tf.while_loop( loop_condition, loop_body, loop_vars=[time_step, init_decoder_state, rnn_output, context_output], shape_invariants=[ tf.TensorShape([]), nest.map_structure(get_state_shape_invariants, init_decoder_state), tf.TensorShape([None, None, conf.hidden_size]), tf.TensorShape([None, None, conf.hidden_size * 2]), ]) # body_output = tf.concat([rnn_output, context_output], axis=-1) # body_output = tf.layers.dense(body_output, self.hidden_size, activation=tf.nn.tanh, use_bias=True, name='body_output_layer') decoder_output = rnn_output logits = proj_logits(decoder_output, conf.embed_size, conf.vocab_size, name='share_embedding') # logits = proj_logits(encoder_output[:,:,:300], conf.embed_size, conf.vocab_size, name='share_embedding') onehot_s2 = tf.one_hot(self.s2, depth=conf.vocab_size) # [batch,len,vocab] xentropy = tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=onehot_s2) # [batch,len] weights = tf.to_float(tf.not_equal(self.s2, 0)) # [batch,len] 1 for nonpad; 0 for pad loss_num = xentropy * weights # [batch,len] loss_den = weights # [batch,len] loss = tf.reduce_sum(loss_num) / tf.reduce_sum(loss_den) # scalar self.loss = loss # self.sent_loss = tf.reduce_sum(loss_num, -1) / tf.reduce_sum(loss_den, -1) # [batch] # decoder infer cache = {'state': decoder_rnn.zero_state(batch_size, tf.float32)} def symbols_to_logits_fn(ids, i, cache): # ids [batch,length] pred_target = ids[:, -1:] # 截取最后一个 [batch,1] embed_target, _ = embedding(tf.expand_dims(pred_target, axis=-1), conf.vocab_size, conf.embed_size, 'share_embedding') # [batch,length,embed] decoder_input = tf.squeeze(embed_target, axis=1) # [batch,embed] # if use attention s = cache['state'] if isinstance(decoder_rnn, tf.nn.rnn_cell.GRUCell) else cache['state'].h context = encdec_atten(s, beam_size=conf.beam_size) # [batch,hidden] decoder_input = tf.concat([decoder_input, context], axis=-1) # [batch,hidden+] # run rnn # with tf.variable_scope('rnn', reuse=tf.AUTO_REUSE): decoder_output, cache['state'] = decoder_rnn(decoder_input, cache['state']) logits = proj_logits(decoder_output, conf.hidden_size, conf.vocab_size, name='share_embedding') return logits, cache initial_ids = tf.zeros([batch_size], dtype=tf.int32) # <pad>为<sos> def greedy_search_wrapper(): """ Greedy Search """ decoded_ids, scores = greedy_search( symbols_to_logits_fn, initial_ids, max_decode_len=conf.max_decode_len, cache=cache, eos_id=conf.eos_id, ) return decoded_ids, scores def beam_search_wrapper(): """ Beam Search """ decoded_ids, scores = beam_search( # [batch,beam,len] [batch,beam] symbols_to_logits_fn, initial_ids, beam_size=conf.beam_size, max_decode_len=conf.max_decode_len, vocab_size=conf.vocab_size, states=cache, eos_id=conf.eos_id, gamma=conf.gamma, num_group=conf.num_group, top_k=conf.top_k, ) return decoded_ids, scores decoded_ids, scores = tf.cond(tf.equal(conf.beam_size, 1), greedy_search_wrapper, beam_search_wrapper) self.decoded_ids = tf.identity(decoded_ids, name='decoded_ids') # [batch,beam/1,len] self.scores = tf.identity(scores, name='scores') # [batch,beam/1] self.global_step = tf.train.get_or_create_global_step() self.optimizer = tf.train.AdamOptimizer(learning_rate=conf.lr) self.train_op = self.optimizer.minimize(self.loss, global_step=self.global_step)
def build_model2(self): # biGRU encoder + bah_attn + GRU decoder # embedding # [batch,len,embed] # pretrained_word_embeddings = np.load(f'{curr_dir}/pretrain_emb_300.npy') pretrained_word_embeddings = None s1_embed, _ = embedding(tf.expand_dims(self.s1, -1), conf.vocab_size, conf.embed_size, name='share_embedding', pretrain_embedding=pretrained_word_embeddings) s1_mask = mask_nonpad_from_embedding(s1_embed) # [batch,len] 1 for nonpad; 0 for pad s1_seqlen = tf.cast(tf.reduce_sum(s1_mask, axis=-1), tf.int32) # [batch] # encoder encoder_input = s1_embed encoder_input = tf.layers.dropout(encoder_input, rate=self.dropout_rate) # dropout with tf.variable_scope('birnn_encoder'): self.bilstm_encoder1 = Bi_RNN(cell_name='GRUCell', hidden_size=conf.hidden_size, dropout_rate=self.dropout_rate) encoder_output, _ = self.bilstm_encoder1(encoder_input, s1_seqlen) # [batch,len,2hid] # decoder s2_embed, _ = embedding(tf.expand_dims(self.s2, -1), conf.vocab_size, conf.embed_size, name='share_embedding', pretrain_embedding=pretrained_word_embeddings) s2_mask = mask_nonpad_from_embedding(s2_embed) # [batch,len] 1 for nonpad; 0 for pad s2_seqlen = tf.cast(tf.reduce_sum(s2_mask, -1), tf.int32) # [batch] decoder_input = s2_embed decoder_input = shift_right(decoder_input) # 用pad当做eos decoder_input = tf.layers.dropout(decoder_input, rate=self.dropout_rate) # dropout decoder_rnn = tf.nn.rnn_cell.DropoutWrapper(getattr(tf.nn.rnn_cell, 'GRUCell')(conf.hidden_size), # GRUCell/LSTMCell input_keep_prob=1.0 - self.dropout_rate) attention_mechanism = getattr(tf.contrib.seq2seq, 'BahdanauAttention')( conf.hidden_size, encoder_output, memory_sequence_length=s1_seqlen, name='BahdanauAttention', ) cell = tf.contrib.seq2seq.AttentionWrapper(decoder_rnn, attention_mechanism, output_attention=False, name='attention_wrapper', ) with tf.variable_scope('decoder'): decoder_output, _ = tf.nn.dynamic_rnn( cell, decoder_input, s2_seqlen, initial_state=None, # 默认用0向量初始化 dtype=tf.float32, time_major=False ) # 默认scope是rnn e.g.decoder/rnn/kernal logits = proj_logits(decoder_output, conf.embed_size, conf.vocab_size, name='share_embedding') # logits = proj_logits(encoder_output[:,:,:300], conf.embed_size, conf.vocab_size, name='share_embedding') onehot_s2 = tf.one_hot(self.s2, depth=conf.vocab_size) # [batch,len,vocab] xentropy = tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=onehot_s2) # [batch,len] weights = tf.to_float(tf.not_equal(self.s2, 0)) # [batch,len] 1 for nonpad; 0 for pad loss_num = xentropy * weights # [batch,len] loss_den = weights # [batch,len] loss = tf.reduce_sum(loss_num) / tf.reduce_sum(loss_den) # scalar self.loss = loss # self.sent_loss = tf.reduce_sum(loss_num, -1) / tf.reduce_sum(loss_den, -1) # [batch] """ decoder infer """ # batch_size = shape_list(encoder_output)[0] batch_size = shape_list(encoder_output)[0] last_dim = shape_list(encoder_output)[-1] tile_encoder_output = tf.tile(tf.expand_dims(encoder_output, 1), [1, conf.beam_size, 1, 1]) tile_encoder_output = tf.reshape(tile_encoder_output, [batch_size * conf.beam_size, -1, last_dim]) tile_s1_seqlen = tf.tile(tf.expand_dims(s1_seqlen, 1), [1, conf.beam_size]) tile_s1_seqlent = tf.reshape(tile_s1_seqlen, [-1]) # 因为tf.BahdanauAttention在初始时就要指定beam_size来tile memory, 所以这里初始化另外一个专用于推断的beam_size tiled的并共享参数 # 不过验证还是有问题,不能用reuse=True 报 Variable memory_layer_1/kernel does not exist, or was not created with tf.get_variable() # 故不建议使用 with tf.variable_scope('', reuse=tf.AUTO_REUSE): attention_mechanism_decoder = getattr(tf.contrib.seq2seq, 'BahdanauAttention')( conf.hidden_size, tile_encoder_output, memory_sequence_length=tile_s1_seqlent, name='BahdanauAttention', ) cell_decoder = tf.contrib.seq2seq.AttentionWrapper(decoder_rnn, attention_mechanism_decoder, output_attention=False, name='attention_wrapper', ) initial_state = cell_decoder.zero_state(batch_size * conf.beam_size, tf.float32) # 内部会检查batch_size与encoder_output是否一致,需乘beam_size # 初始化缓存 # 区分能否设在cache: cache的值在beam_search过程中会expand和merge,需要tensor rank大于1 cache = { 'cell_state': initial_state.cell_state, 'attention': initial_state.attention, 'alignments': initial_state.alignments, 'attention_state': initial_state.attention_state, } unable_cache = { 'alignment_history': initial_state.alignment_history, # 'time': initial_state.time } # 将cache先变回batch,beam_search过程会expand/merge/gather,使得state是符合batch*beam的 cache = nest.map_structure(lambda s: s[:batch_size], cache) def symbols_to_logits_fn(ids, i, cache): nonlocal unable_cache ids = ids[:, -1:] target = tf.expand_dims(ids, axis=-1) # [batch,1,1] embedding_target, _ = embedding(target, conf.vocab_size, conf.hidden_size, 'share_embedding', reuse=True) input = tf.squeeze(embedding_target, axis=1) # [batch,hid] # 合并 cache和unable_cache为state state = cell_decoder.zero_state(batch_size * conf.beam_size, tf.float32).clone( cell_state=cache['cell_state'], attention=cache['attention'], alignments=cache['alignments'], attention_state=cache['attention_state'], alignment_history=unable_cache['alignment_history'], # time=unable_cache['time'], time=tf.convert_to_tensor(i, dtype=tf.int32), ) with tf.variable_scope('decoder/rnn', reuse=tf.AUTO_REUSE): output, state = cell_decoder(input, state) # 分开cache和unable_cache cache['cell_state'] = state.cell_state cache['attention'] = state.attention cache['alignments'] = state.alignments cache['attention_state'] = state.attention_state unable_cache['alignment_history'] = state.alignment_history # unable_cache['time'] = state.time body_output = output # [batch,hidden] logits = proj_logits(body_output, conf.embed_size, conf.vocab_size, name='share_embedding') return logits, cache initial_ids = tf.zeros([batch_size], dtype=tf.int32) # <pad>为<sos> def greedy_search_wrapper(): """ Greedy Search """ decoded_ids, scores = greedy_search( symbols_to_logits_fn, initial_ids, max_decode_len=conf.max_decode_len, cache=cache, eos_id=conf.eos_id, ) return decoded_ids, scores def beam_search_wrapper(): """ Beam Search """ decoded_ids, scores = beam_search( # [batch,beam,len] [batch,beam] symbols_to_logits_fn, initial_ids, conf.beam_size, conf.max_decode_len, conf.vocab_size, states=cache, eos_id=conf.eos_id, gamma=conf.gamma, num_group=conf.num_group, top_k=conf.top_k, ) return decoded_ids, scores decoded_ids, scores = tf.cond(tf.equal(conf.beam_size, 1), greedy_search_wrapper, beam_search_wrapper) self.decoded_ids = tf.identity(decoded_ids, name='decoded_ids') # [batch,beam/1,len] self.scores = tf.identity(scores, name='scores') # [batch,beam/1] self.global_step = tf.train.get_or_create_global_step() self.optimizer = tf.train.AdamOptimizer(learning_rate=conf.lr) self.train_op = self.optimizer.minimize(self.loss, global_step=self.global_step)
def build_model1(self): # embedding # [batch,len,embed] s1_embed, _ = embedding(tf.expand_dims(self.s1, -1), conf.vocab_size, conf.embed_size, name='embedding', pretrain_embedding=conf.pretrain_emb) s1_mask = mask_nonpad_from_embedding( s1_embed) # [batch,len] 1 for nonpad; 0 for pad s1_seqlen = tf.cast(tf.reduce_sum(s1_mask, axis=-1), tf.int32) # [batch] # encoder encoder_input = s1_embed encoder_input = tf.layers.dropout(encoder_input, rate=self.dropout_rate) # dropout with tf.variable_scope('rnn_1'): self.bilstm_encoder1 = Bi_RNN(cell_name='LSTMCell', hidden_size=conf.birnn_hidden_size, dropout_rate=self.dropout_rate) encoder_output, _ = self.bilstm_encoder1( encoder_input, s1_seqlen) # [batch,len,2hid] with tf.variable_scope('rnn_2'): self.bilstm_encoder2 = Bi_RNN(cell_name='LSTMCell', hidden_size=conf.birnn_hidden_size, dropout_rate=self.dropout_rate) encoder_output, _ = self.bilstm_encoder2( encoder_output, s1_seqlen) # [batch,len,2hid] # logits cls_logits = tf.layers.dense(encoder_output, conf.label_size, activation=None, use_bias=True, name='cls_logits') # [batch,length,label] # crf log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood( cls_logits, self.ner_label, tf.to_int32(s1_seqlen)) crf_loss = tf.reduce_mean(-log_likelihood) # loss self.loss = crf_loss # crf decode ner_pred, ner_prob = tf.contrib.crf.crf_decode(cls_logits, transition_params, tf.to_int32(s1_seqlen)) self.ner_prob = tf.identity(ner_prob, name='ner_prob') # [batch] self.ner_pred = tf.identity(ner_pred, name='ner_pred') # [batch,len] with tf.name_scope('accuracy'): ner_acc = tf.cast(tf.equal(self.ner_pred, self.ner_label), tf.float32) * s1_mask # [batch,length] ner_acc = tf.reduce_sum(ner_acc, axis=-1) # [batch] ner_acc = ner_acc / tf.cast(s1_seqlen, tf.float32) # [batch] self.accuracy = tf.reduce_mean(ner_acc, axis=0) # scalar self.global_step = tf.train.get_or_create_global_step() self.optimizer = tf.train.AdamOptimizer(learning_rate=conf.lr) self.train_op = self.optimizer.minimize(self.loss, global_step=self.global_step)