def beam_search_decode_with_ctc(x, han_vocab, sess): enc, lpz = sess.run([return_tensors[3], return_tensors[4]], {return_tensors[0]: np.array([x])}) lpz = np.squeeze(lpz, axis=0) if ctc_weight == 0.0: lpz =None maxlen = enc.shape[1] minlen = 0 eos = han_vocab.index('<EOS>') hyp = {'score': 0.0, 'yseq': [eos]} if lpz is not None: ctc_prefix_score = CTCPrefixScore(lpz, han_vocab.index('*'), eos) hyp['ctc_state_prev'] = ctc_prefix_score.initial_state() hyp['ctc_score_prev'] = 0.0 # print(hyp['ctc_state_prev']) if ctc_weight != 1.0: ctc_beam = min(lpz.shape[-1], int(beam * CTC_SCORING_RATIO)) else: ctc_beam = lpz.shape[-1] hyps = [hyp] ended_hyps = [] for i in range(maxlen): hyps_best_kept = [] max_len = max(len(hyp['yseq']) for hyp in hyps) ys = np.ones(shape=(len(hyps),max_len))*eos for k, hyp in enumerate(hyps): ys[k, :len(hyp['yseq'])] =hyp['yseq'] local_att_scores_all = sess.run(return_tensors[5], {return_tensors[3]: np.tile(enc,(len(hyps),1,1)), return_tensors[1]: ys, return_tensors[2]: np.tile(np.array([[maxlen]]),(len(hyps),1))}) # * 是ctc占位符 语言模型中不会出现 local_att_scores_all[:, han_vocab.index('*')] = -10000000000.0 local_scores_all = local_att_scores_all if lpz is not None: local_best_scores_all, local_best_ids_all = sess.run([score, ids], {topk_input: local_att_scores_all, beam_size: ctc_beam}) ctc_state_prev = np.array([hyp['ctc_state_prev'] for hyp in hyps]) ctc_score_prev = np.array([hyp['ctc_score_prev'] for hyp in hyps]) ctc_score_prev = ctc_score_prev[:, np.newaxis] ctc_scores_all, ctc_states_all = ctc_prefix_score( ys, local_best_ids_all, ctc_state_prev) local_att_scores_all_now = local_att_scores_all[ np.arange(0, local_best_ids_all.shape[0])[:, np.newaxis], local_best_ids_all] local_scores_all = (1.0 - ctc_weight) * local_att_scores_all_now \ + ctc_weight * (ctc_scores_all - ctc_score_prev) local_best_scores_all, joint_best_ids_all = sess.run([score, ids], {topk_input: local_scores_all, beam_size: beam}) local_best_ids_all = local_best_ids_all[ np.arange(0, joint_best_ids_all.shape[0])[:, np.newaxis], joint_best_ids_all] else: local_best_scores_all, local_best_ids_all = sess.run([score, ids], {topk_input: local_scores_all, beam_size: beam}) for k, hyp in enumerate(hyps): for j in range(beam): new_hyp = {} new_hyp['score'] = hyp['score'] + float(local_best_scores_all[k, j]) new_hyp['yseq'] = [0] * (1 + len(hyp['yseq'])) new_hyp['yseq'][:len(hyp['yseq'])] = hyp['yseq'] new_hyp['yseq'][len(hyp['yseq'])] = int(local_best_ids_all[k, j]) if lpz is not None: new_hyp['ctc_state_prev'] = ctc_states_all[k, joint_best_ids_all[k, j]] new_hyp['ctc_score_prev'] = ctc_scores_all[k, joint_best_ids_all[k, j]] hyps_best_kept.append(new_hyp) hyps_best_kept = sorted( hyps_best_kept, key=lambda x: x['score'], reverse=True)[:beam] hyps = hyps_best_kept if i == maxlen - 1: for hyp in hyps: if hyp['yseq'][-1] != eos: hyp['yseq'].append(eos) remained_hyps = [] for hyp in hyps: if hyp['yseq'][-1] == eos: if len(hyp['yseq']) > minlen: hyp['score'] += (i+1) * penalty ended_hyps.append(hyp) else: remained_hyps.append(hyp) if end_detect(ended_hyps, i): break hyps = remained_hyps if len(hyps) <= 0: break ended_hyps=sorted( ended_hyps, key=lambda x: x['score'], reverse=True) lenth = len(ended_hyps) if lenth == 0: return [] index_l = [0] index_r = [len(ended_hyps)] for i in range(1,lenth): diff = len(ended_hyps[i]['yseq'])-len(ended_hyps[i-1]['yseq']) # 相邻分值差不多的 应该字数差不多 干掉字少的 if abs(diff) > 3: if diff < 0: index_r.append(i) else: index_l.append(i) # 相邻分值相差很大 干掉前面分值大的 越大字越少 elif ended_hyps[i]['score'] - ended_hyps[i - 1]['score'] < -10: index_r.append(i) m_l = max(index_l) m_r = min(index_r) if m_l >= m_r: m_r = lenth ended_hyps = ended_hyps[m_l:m_r] a_s = [i['score'] for i in ended_hyps] a_seq = [[han_vocab[b] for b in i['yseq']] for i in ended_hyps] for a, b in zip(a_s, a_seq): print(a, b) nbest_hyps = sorted( ended_hyps, key=lambda x: x['score'], reverse=True)[:min(len(ended_hyps), nbest)] if len(nbest_hyps) == 0: return [] result = [han_vocab[i] for i in nbest_hyps[0]['yseq'][1:-1]] return result
def recognize_beam(self, h, lpz, recog_args, char_list, rnnlm=None): '''beam search implementation :param h: :param recog_args: :param char_list: :return: ''' logging.info('input lengths: ' + str(h.shape[0])) # initialization c_list = [None] # list of cell state of each layer z_list = [None] # list of hidden state of each layer for l in six.moves.range(1, self.dlayers): c_list.append(None) z_list.append(None) a = None self.att.reset() # reset pre-computation of h # search parms beam = recog_args.beam_size penalty = recog_args.penalty ctc_weight = recog_args.ctc_weight # preprate sos y = self.xp.full(1, self.sos, 'i') if recog_args.maxlenratio == 0: maxlen = h.shape[0] else: # maxlen >= 1 maxlen = max(1, int(recog_args.maxlenratio * h.shape[0])) minlen = int(recog_args.minlenratio * h.shape[0]) logging.info('max output length: ' + str(maxlen)) logging.info('min output length: ' + str(minlen)) # initialize hypothesis if rnnlm: hyp = {'score': 0.0, 'yseq': [y], 'c_prev': c_list, 'z_prev': z_list, 'a_prev': a, 'rnnlm_prev': None} else: hyp = {'score': 0.0, 'yseq': [y], 'c_prev': c_list, 'z_prev': z_list, 'a_prev': a} if lpz is not None: ctc_prefix_score = CTCPrefixScore(lpz, 0, self.eos, self.xp) hyp['ctc_state_prev'] = ctc_prefix_score.initial_state() hyp['ctc_score_prev'] = 0.0 if ctc_weight != 1.0: # pre-pruning based on attention scores ctc_beam = min(lpz.shape[-1], int(beam * CTC_SCORING_RATIO)) else: ctc_beam = lpz.shape[-1] hyps = [hyp] ended_hyps = [] for i in six.moves.range(maxlen): logging.debug('position ' + str(i)) hyps_best_kept = [] for hyp in hyps: ey = self.embed(hyp['yseq'][i]) # utt list (1) x zdim att_c, att_w = self.att([h], hyp['z_prev'][0], hyp['a_prev']) ey = F.hstack((ey, att_c)) # utt(1) x (zdim + hdim) c_list[0], z_list[0] = self.lstm0(hyp['c_prev'][0], hyp['z_prev'][0], ey) for l in six.moves.range(1, self.dlayers): c_list[l], z_list[l] = self['lstm%d' % l]( hyp['c_prev'][l], hyp['z_prev'][l], z_list[l - 1]) # get nbest local scores and their ids local_att_scores = F.log_softmax(self.output(z_list[-1])).data if rnnlm: rnnlm_state, local_lm_scores = rnnlm.predict(hyp['rnnlm_prev'], hyp['yseq'][i]) local_scores = local_att_scores + recog_args.lm_weight * local_lm_scores else: local_scores = local_att_scores if lpz is not None: local_best_ids = self.xp.argsort(local_scores, axis=1)[0, ::-1][:ctc_beam] ctc_scores, ctc_states = ctc_prefix_score(hyp['yseq'], local_best_ids, hyp['ctc_state_prev']) local_scores = \ (1.0 - ctc_weight) * local_att_scores[:, local_best_ids] \ + ctc_weight * (ctc_scores - hyp['ctc_score_prev']) if rnnlm: local_scores += recog_args.lm_weight * local_lm_scores[:, local_best_ids] joint_best_ids = self.xp.argsort(local_scores, axis=1)[0, ::-1][:beam] local_best_scores = local_scores[:, joint_best_ids] local_best_ids = local_best_ids[joint_best_ids] else: local_best_ids = self.xp.argsort(local_scores, axis=1)[0, ::-1][:beam] local_best_scores = local_scores[:, local_best_ids] for j in six.moves.range(beam): new_hyp = {} # do not copy {z,c}_list directly new_hyp['z_prev'] = z_list[:] new_hyp['c_prev'] = c_list[:] new_hyp['a_prev'] = att_w new_hyp['score'] = hyp['score'] + local_best_scores[0, j] new_hyp['yseq'] = [0] * (1 + len(hyp['yseq'])) new_hyp['yseq'][:len(hyp['yseq'])] = hyp['yseq'] new_hyp['yseq'][len(hyp['yseq'])] = self.xp.full( 1, local_best_ids[j], 'i') if rnnlm: new_hyp['rnnlm_prev'] = rnnlm_state if lpz is not None: new_hyp['ctc_state_prev'] = ctc_states[joint_best_ids[j]] new_hyp['ctc_score_prev'] = ctc_scores[joint_best_ids[j]] # will be (2 x beam) hyps at most hyps_best_kept.append(new_hyp) hyps_best_kept = sorted( hyps_best_kept, key=lambda x: x['score'], reverse=True)[:beam] # sort and get nbest hyps = hyps_best_kept logging.debug('number of pruned hypothes: ' + str(len(hyps))) logging.debug('best hypo: ' + ''.join([char_list[int(x)] for x in hyps[0]['yseq'][1:]]).replace('<space>', ' ')) # add eos in the final loop to avoid that there are no ended hyps if i == maxlen - 1: logging.info('adding <eos> in the last postion in the loop') for hyp in hyps: hyp['yseq'].append(self.xp.full(1, self.eos, 'i')) # add ended hypothes to a final list, and removed them from current hypothes # (this will be a problem, number of hyps < beam) remained_hyps = [] for hyp in hyps: if hyp['yseq'][-1] == self.eos: # only store the sequence that has more than minlen outputs # also add penalty if len(hyp['yseq']) > minlen: hyp['score'] += (i + 1) * penalty if rnnlm: # Word LM needs to add final <eos> score hyp['score'] += recog_args.lm_weight * rnnlm.final( hyp['rnnlm_prev']) ended_hyps.append(hyp) else: remained_hyps.append(hyp) # end detection if end_detect(ended_hyps, i) and recog_args.maxlenratio == 0.0: logging.info('end detected at %d', i) break hyps = remained_hyps if len(hyps) > 0: logging.debug('remeined hypothes: ' + str(len(hyps))) else: logging.info('no hypothesis. Finish decoding.') break for hyp in hyps: logging.debug('hypo: ' + ''.join([char_list[int(x)] for x in hyp['yseq'][1:]]).replace('<space>', ' ')) logging.debug('number of ended hypothes: ' + str(len(ended_hyps))) nbest_hyps = sorted( ended_hyps, key=lambda x: x['score'], reverse=True)[:min(len(ended_hyps), recog_args.nbest)] # check number of hypotheis if len(nbest_hyps) == 0: logging.warn('there is no N-best results, perform recognition again with smaller minlenratio.') # should copy becasuse Namespace will be overwritten globally recog_args = Namespace(**vars(recog_args)) recog_args.minlenratio = max(0.0, recog_args.minlenratio - 0.1) return self.recognize_beam(h, lpz, recog_args, char_list, rnnlm) logging.info('total log probability: ' + str(nbest_hyps[0]['score'])) logging.info('normalized log probability: ' + str(nbest_hyps[0]['score'] / len(nbest_hyps[0]['yseq']))) return nbest_hyps
def recognize_beam(self, encoder_outputs, char_list, lpz, aligns_pad, args): """Beam search, decode one utterence now. Args: encoder_outputs: T x H char_list: list of character args: args.beam Returns: nbest_hyps: """ # search params #import pdb #pdb.set_trace() beam = args.beam_size nbest = args.nbest ctc_weight = args.ctc_weight CTC_SCORING_RATIO = 1.5 if args.decode_max_len != 0: maxlen = args.decode_max_len elif lpz is not None: maxlen = int(len(torch.nonzero(torch.max(lpz, dim=-1)[1])) * 1.5) elif args.align_trun: maxlen = int(aligns_pad.size(1) * 1.5) # *********Init decoder rnn h_list = [self.zero_state(encoder_outputs.unsqueeze(0))] c_list = [self.zero_state(encoder_outputs.unsqueeze(0))] for l in range(1, self.num_layers): h_list.append(self.zero_state(encoder_outputs.unsqueeze(0))) c_list.append(self.zero_state(encoder_outputs.unsqueeze(0))) att_c = self.zero_state(encoder_outputs.unsqueeze(0), H=encoder_outputs.unsqueeze(0).size(2)) # prepare sos y = self.sos_id vy = encoder_outputs.new_zeros(1).long() hyp = { 'score': 0.0, 'yseq': [y], 'c_prev': c_list, 'h_prev': h_list, 'a_prev': att_c } if lpz is not None: #import pdb #pdb.set_trace() ctc_prefix_score = CTCPrefixScore(lpz.detach().cpu().numpy(), 0, self.eos_id, np) hyp['ctc_state_prev'] = ctc_prefix_score.initial_state() hyp['ctc_score_prev'] = 0.0 if ctc_weight != 1.0: ctc_beam = min(lpz.shape[-1], int(beam * CTC_SCORING_RATIO)) else: ctc_beam = lpz.shape[-1] if args.trun: ctc_greedy = torch.max(lpz, dim=-1)[1].unsqueeze(dim=0) #print(ctc_greedy) aligns = [] for k in range(ctc_greedy.size()[0]): align = (torch.nonzero(ctc_greedy[k]) + 1).reshape(-1).cpu().numpy().tolist() align.insert(0, 0) aligns.append(align) #print(aligns[0:2]) #print(np.shape(aligns)) #aligns = torch.Tensor(aligns).long().cuda() aligns_pad = pad_list([torch.Tensor(y).long() for y in aligns], IGNORE_ID) hyps = [hyp] ended_hyps = [] for i in range(maxlen): hyps_best_kept = [] for hyp in hyps: # vy.unsqueeze(1) vy[0] = hyp['yseq'][i] embedded = self.embedding(vy) # embedded.unsqueeze(0) # step 1. decoder RNN: s_i = RNN(s_i−1,y_i−1,c_i−1) rnn_input = torch.cat((embedded, hyp['a_prev']), dim=1) h_list[0], c_list[0] = self.rnn[0]( rnn_input, (hyp['h_prev'][0], hyp['c_prev'][0])) for l in range(1, self.num_layers): h_list[l], c_list[l] = self.rnn[l]( h_list[l - 1], (hyp['h_prev'][l], hyp['c_prev'][l])) rnn_output = h_list[-1] # step 2. attention: c_i = AttentionContext(s_i,h) # below unsqueeze: (N x H) -> (N x 1 x H) #import pdb #pdb.set_trace() mask = None if args.trun or args.align_trun: mask = torch.ones(encoder_outputs.unsqueeze(0).size(0), encoder_outputs.unsqueeze(0).size(1), dtype=torch.uint8).cuda() #mask = torch.zeros(encoder_outputs.unsqueeze(0).size(0),encoder_outputs.unsqueeze(0).size(1),dtype=torch.uint8).cuda() if i + 1 < aligns_pad.size(1): for m in range(mask.size(0)): if self.peak_left != 0: left_id = max(i - self.peak_left + 1, 0) else: left_id = 0 right_id = min(i + 1 + self.peak_right, aligns_pad.size(1) - 1) left_bound = min( aligns_pad[m][left_id] + self.offset, rnn_output.size(1)) right_bound = max( min(aligns_pad[m][right_id] + self.offset, rnn_output.size(1)), 0) #right_bound = max(min(aligns_pad[m][i+1] + self.offset, rnn_output.size(1)), 0) #left_bound = 0 #mask[m][0:right_bound] = 0 #mask[m][right_bound:-1] = 1 mask[m][left_bound:right_bound] = 0 att_c, att_w = self.attention(rnn_output.unsqueeze(dim=1), encoder_outputs.unsqueeze(0), mask) att_c = att_c.squeeze(dim=1) # step 3. concate s_i and c_i, and input to MLP mlp_input = torch.cat((rnn_output, att_c), dim=1) predicted_y_t = self.mlp(mlp_input) local_att_scores = F.log_softmax(predicted_y_t, dim=1) local_scores = local_att_scores if args.ctc_weight > 0: #import pdb #pdb.set_trace() local_best_scores, local_best_ids = torch.topk( local_att_scores, ctc_beam, dim=1) ctc_scores, ctc_states = ctc_prefix_score( hyp['yseq'], local_best_ids[0], hyp['ctc_state_prev']) local_scores = ( 1.0 - ctc_weight) * local_att_scores[:, local_best_ids[ 0]] + ctc_weight * torch.from_numpy( ctc_scores - hyp['ctc_score_prev']).cuda() local_best_scores, joint_best_ids = torch.topk( local_scores, beam, dim=1) local_best_ids = local_best_ids[:, joint_best_ids[0]] else: # topk scores local_best_scores, local_best_ids = torch.topk( local_scores, beam, dim=1) for j in range(beam): new_hyp = {} new_hyp['h_prev'] = h_list[:] new_hyp['c_prev'] = c_list[:] new_hyp['a_prev'] = att_c[:] new_hyp['score'] = hyp['score'] + local_best_scores[0, j] new_hyp['yseq'] = [0] * (1 + len(hyp['yseq'])) new_hyp['yseq'][:len(hyp['yseq'])] = hyp['yseq'] new_hyp['yseq'][len(hyp['yseq'])] = int(local_best_ids[0, j]) # will be (2 x beam) hyps at most if args.ctc_weight > 0: new_hyp['ctc_state_prev'] = ctc_states[joint_best_ids[ 0, j]] new_hyp['ctc_score_prev'] = ctc_scores[joint_best_ids[ 0, j]] hyps_best_kept.append(new_hyp) hyps_best_kept = sorted(hyps_best_kept, key=lambda x: x['score'], reverse=True)[:beam] # end for hyp in hyps hyps = hyps_best_kept # add eos in the final loop to avoid that there are no ended hyps if i == maxlen - 1: for hyp in hyps: hyp['yseq'].append(self.eos_id) # add ended hypothes to a final list, and removed them from current hypothes # (this will be a probmlem, number of hyps < beam) remained_hyps = [] for hyp in hyps: if hyp['yseq'][-1] == self.eos_id: # hyp['score'] += (i + 1) * penalty ended_hyps.append(hyp) else: remained_hyps.append(hyp) hyps = remained_hyps if len(hyps) > 0: print('remeined hypothes: ' + str(len(hyps))) else: print('no hypothesis. Finish decoding.') break #import pdb #pdb.set_trace() for hyp in hyps: print('hypo: ' + ' '.join([char_list[int(x)] for x in hyp['yseq'][1:]])) # end for i in range(maxlen) nbest_hyps = sorted(ended_hyps, key=lambda x: x['score'], reverse=True)[:min(len(ended_hyps), nbest)] #print(nbest_hyps) return nbest_hyps
def recognize_beam(self, h, lpz, recog_args, char_list): '''beam search implementation :param Variable h: :param Namespace recog_args: :param char_list: :return: ''' logging.info('input lengths: ' + str(h.size(0))) # initialization c_list = [self.zero_state(h.unsqueeze(0))] z_list = [self.zero_state(h.unsqueeze(0))] for l in six.moves.range(1, self.dlayers): c_list.append(self.zero_state(h.unsqueeze(0))) z_list.append(self.zero_state(h.unsqueeze(0))) a = None self.att.reset() # reset pre-computation of h # search parms beam = recog_args.beam_size penalty = recog_args.penalty ctc_weight = recog_args.ctc_weight # preprate sos y = self.sos vy = Variable(h.data.new(1).zero_().long(), volatile=True) if recog_args.maxlenratio == 0: maxlen = h.shape[0] else: # maxlen >= 1 maxlen = max(1, int(recog_args.maxlenratio * h.size(0))) minlen = int(recog_args.minlenratio * h.size(0)) logging.info('max output length: ' + str(maxlen)) logging.info('min output length: ' + str(minlen)) # initialize hypothesis hyp = { 'score': 0.0, 'yseq': [y], 'c_prev': c_list, 'z_prev': z_list, 'a_prev': a } if lpz is not None: ctc_prefix_score = CTCPrefixScore(lpz.numpy(), 0, self.eos, np) hyp['ctc_state_prev'] = ctc_prefix_score.initial_state() hyp['ctc_score_prev'] = 0.0 ctc_beam = min(lpz.shape[-1], int(beam * CTC_SCORING_RATIO)) hyps = [hyp] ended_hyps = [] for i in six.moves.range(maxlen): logging.debug('position ' + str(i)) hyps_best_kept = [] for hyp in hyps: vy.unsqueeze(1) vy[0] = hyp['yseq'][i] ey = self.embed(vy) # utt list (1) x zdim ey.unsqueeze(0) att_c, att_w = self.att(h.unsqueeze(0), [h.size(0)], hyp['z_prev'][0], hyp['a_prev']) ey = torch.cat((ey, att_c), dim=1) # utt(1) x (zdim + hdim) z_list[0], c_list[0] = self.decoder[0]( ey, (hyp['z_prev'][0], hyp['c_prev'][0])) for l in six.moves.range(1, self.dlayers): z_list[l], c_list[l] = self.decoder[l]( z_list[l - 1], (hyp['z_prev'][l], hyp['c_prev'][l])) # get nbest local scores and their ids local_att_scores = F.log_softmax(self.output(z_list[-1]), dim=1).data if lpz is not None: local_best_scores, local_best_ids = torch.topk( local_att_scores, ctc_beam, dim=1) ctc_scores, ctc_states = ctc_prefix_score( hyp['yseq'], local_best_ids[0], hyp['ctc_state_prev']) local_scores = \ (1.0 - ctc_weight) * local_att_scores[:, local_best_ids[0]] \ + ctc_weight * torch.from_numpy(ctc_scores - hyp['ctc_score_prev']) local_best_scores, joint_best_ids = torch.topk( local_scores, beam, dim=1) local_best_ids = local_best_ids[:, joint_best_ids[0]] else: local_best_scores, local_best_ids = torch.topk( local_att_scores, beam, dim=1) for j in six.moves.range(beam): new_hyp = {} new_hyp['z_prev'] = [z_list[0]] new_hyp['c_prev'] = [c_list[0]] for l in six.moves.range(1, self.dlayers): new_hyp['z_prev'].append(z_list[l]) new_hyp['c_prev'].append(c_list[l]) new_hyp['a_prev'] = att_w new_hyp['score'] = hyp['score'] + local_best_scores[0, j] new_hyp['yseq'] = [0] * (1 + len(hyp['yseq'])) new_hyp['yseq'][:len(hyp['yseq'])] = hyp['yseq'] new_hyp['yseq'][len(hyp['yseq'])] = local_best_ids[0, j] if lpz is not None: new_hyp['ctc_state_prev'] = ctc_states[joint_best_ids[ 0, j]] new_hyp['ctc_score_prev'] = ctc_scores[joint_best_ids[ 0, j]] # will be (2 x beam) hyps at most hyps_best_kept.append(new_hyp) hyps_best_kept = sorted(hyps_best_kept, key=lambda x: x['score'], reverse=True)[:beam] # sort and get nbest hyps = hyps_best_kept logging.debug('number of pruned hypothes: ' + str(len(hyps))) logging.debug( 'best hypo: ' + ''.join([char_list[int(x)] for x in hyps[0]['yseq'][1:]])) # add eos in the final loop to avoid that there are no ended hyps if i == maxlen - 1: logging.info('adding <eos> in the last postion in the loop') for hyp in hyps: hyp['yseq'].append(self.eos) # add ended hypothes to a final list, and removed them from current hypothes # (this will be a probmlem, number of hyps < beam) remained_hyps = [] for hyp in hyps: if hyp['yseq'][-1] == self.eos: # only store the sequence that has more than minlen outputs # also add penalty if len(hyp['yseq']) > minlen: hyp['score'] += (i + 1) * penalty ended_hyps.append(hyp) else: remained_hyps.append(hyp) # end detection if end_detect(ended_hyps, i) and recog_args.maxlenratio == 0.0: logging.info('end detected at %d', i) break hyps = remained_hyps if len(hyps) > 0: logging.debug('remeined hypothes: ' + str(len(hyps))) else: logging.info('no hypothesis. Finish decoding.') break for hyp in hyps: logging.debug( 'hypo: ' + ''.join([char_list[int(x)] for x in hyp['yseq'][1:]])) logging.debug('number of ended hypothes: ' + str(len(ended_hyps))) best_hyp = sorted(ended_hyps, key=lambda x: x['score'], reverse=True)[0] logging.info('total log probability: ' + str(best_hyp['score'])) logging.info('normalized log probability: ' + str(best_hyp['score'] / len(best_hyp['yseq']))) # remove sos return best_hyp['yseq'][1:]
def recognize_beam(self, encoder_outputs, char_list, lpz, args): """Beam search, decode one utterence now. Args: encoder_outputs: T x H char_list: list of character args: args.beam Returns: nbest_hyps: """ # search params beam = args.beam_size nbest = args.nbest ctc_weight = args.ctc_weight CTC_SCORING_RATIO = 1.5 if args.decode_max_len == 0: maxlen = encoder_outputs.size(0) else: maxlen = args.decode_max_len # *********Init decoder rnn h_list = [self.zero_state(encoder_outputs.unsqueeze(0))] c_list = [self.zero_state(encoder_outputs.unsqueeze(0))] for l in range(1, self.num_layers): h_list.append(self.zero_state(encoder_outputs.unsqueeze(0))) c_list.append(self.zero_state(encoder_outputs.unsqueeze(0))) att_c = self.zero_state(encoder_outputs.unsqueeze(0), H=encoder_outputs.unsqueeze(0).size(2)) # prepare sos y = self.sos_id vy = encoder_outputs.new_zeros(1).long() hyp = { 'score': 0.0, 'yseq': [y], 'c_prev': c_list, 'h_prev': h_list, 'a_prev': att_c } if lpz is not None: import pdb pdb.set_trace() ctc_prefix_score = CTCPrefixScore(lpz.detach().cpu().numpy(), 0, self.eos_id, np) hyp['ctc_state_prev'] = ctc_prefix_score.initial_state() hyp['ctc_score_prev'] = 0.0 if ctc_weight != 1.0: ctc_beam = min(lpz.shape[-1], int(beam * CTC_SCORING_RATIO)) else: ctc_beam = lpz.shape[-1] hyps = [hyp] ended_hyps = [] for i in range(maxlen): hyps_best_kept = [] for hyp in hyps: # vy.unsqueeze(1) vy[0] = hyp['yseq'][i] embedded = self.embedding(vy) # embedded.unsqueeze(0) # step 1. decoder RNN: s_i = RNN(s_i−1,y_i−1,c_i−1) rnn_input = torch.cat((embedded, hyp['a_prev']), dim=1) h_list[0], c_list[0] = self.rnn[0]( rnn_input, (hyp['h_prev'][0], hyp['c_prev'][0])) for l in range(1, self.num_layers): h_list[l], c_list[l] = self.rnn[l]( h_list[l - 1], (hyp['h_prev'][l], hyp['c_prev'][l])) rnn_output = h_list[-1] # step 2. attention: c_i = AttentionContext(s_i,h) # below unsqueeze: (N x H) -> (N x 1 x H) att_c, att_w = self.attention(rnn_output.unsqueeze(dim=1), encoder_outputs.unsqueeze(0)) att_c = att_c.squeeze(dim=1) # step 3. concate s_i and c_i, and input to MLP mlp_input = torch.cat((rnn_output, att_c), dim=1) predicted_y_t = self.mlp(mlp_input) local_att_scores = F.log_softmax(predicted_y_t, dim=1) local_scores = local_att_scores if lpz is not None: #import pdb #pdb.set_trace() local_best_scores, local_best_ids = torch.topk( local_att_scores, ctc_beam, dim=1) ctc_scores, ctc_states = ctc_prefix_score( hyp['yseq'], local_best_ids[0], hyp['ctc_state_prev']) local_scores = ( 1.0 - ctc_weight) * local_att_scores[:, local_best_ids[ 0]] + ctc_weight * torch.from_numpy( ctc_scores - hyp['ctc_score_prev']).cuda() local_best_scores, joint_best_ids = torch.topk( local_scores, beam, dim=1) local_best_ids = local_best_ids[:, joint_best_ids[0]] else: # topk scores local_best_scores, local_best_ids = torch.topk( local_scores, beam, dim=1) for j in range(beam): new_hyp = {} new_hyp['h_prev'] = h_list[:] new_hyp['c_prev'] = c_list[:] new_hyp['a_prev'] = att_c[:] new_hyp['score'] = hyp['score'] + local_best_scores[0, j] new_hyp['yseq'] = [0] * (1 + len(hyp['yseq'])) new_hyp['yseq'][:len(hyp['yseq'])] = hyp['yseq'] new_hyp['yseq'][len(hyp['yseq'])] = int(local_best_ids[0, j]) # will be (2 x beam) hyps at most if lpz is not None: new_hyp['ctc_state_prev'] = ctc_states[joint_best_ids[ 0, j]] new_hyp['ctc_score_prev'] = ctc_scores[joint_best_ids[ 0, j]] hyps_best_kept.append(new_hyp) hyps_best_kept = sorted(hyps_best_kept, key=lambda x: x['score'], reverse=True)[:beam] # end for hyp in hyps hyps = hyps_best_kept # add eos in the final loop to avoid that there are no ended hyps if i == maxlen - 1: for hyp in hyps: hyp['yseq'].append(self.eos_id) # add ended hypothes to a final list, and removed them from current hypothes # (this will be a probmlem, number of hyps < beam) remained_hyps = [] for hyp in hyps: if hyp['yseq'][-1] == self.eos_id: # hyp['score'] += (i + 1) * penalty ended_hyps.append(hyp) else: remained_hyps.append(hyp) hyps = remained_hyps if len(hyps) > 0: print('remeined hypothes: ' + str(len(hyps))) else: print('no hypothesis. Finish decoding.') break for hyp in hyps: print('hypo: ' + ' '.join([char_list[int(x)] for x in hyp['yseq'][1:]])) # end for i in range(maxlen) nbest_hyps = sorted(ended_hyps, key=lambda x: x['score'], reverse=True)[:min(len(ended_hyps), nbest)] #print(nbest_hyps) return nbest_hyps