def matching_to_labeled_hypothesis(self, hypothesis, matching):
""" Helper function for constructing the GT for a given input. """
# matching is a list of (gt_idx, iou_list, bbox_list, label_list)
labeled_hypothesis = Hypothesis(tracklet=hypothesis.tracklet)
if len(matching) == 0:
labeled_hypothesis.labels =\
np.zeros((self.scene_size,), dtype=np.float32)
labeled_hypothesis.bboxes = \
np.zeros((self.scene_size, 4), dtype=np.float32)
labeled_hypothesis.ious = \
np.zeros((self.scene_size,), dtype=np.float32)
labeled_hypothesis.gt_idx = -1
return labeled_hypothesis
idfs = [sum(m[1]) for m in matching]
best_match = np.argmax(np.asarray(idfs))
gt_idx = matching[best_match][0]
labeled_hypothesis.ious = matching[best_match][1]
labeled_hypothesis.bboxes = matching[best_match][2]
labeled_hypothesis.labels = matching[best_match][3]
labeled_hypothesis.gt_idx = gt_idx
return labeled_hypothesis
def convert_hypothesis(pred_sent, vocab, score):
tgt_sent = []
for word_id in pred_sent[1:-1]:
tgt_sent.append(vocab.tgt.id2word[word_id.item()])
if decoder_config.greedy_search:
score = (score - np.log(len(pred_sent)))
return Hypothesis(tgt_sent, score)
def beam_search(self,
src_sent,
key,
max_step=None,
replace=False) -> List[Hypothesis]:
src_sent, key
np_sent = np.array([self.vocab_src[key][word] for word in src_sent])
# if config.flip_source:
# np_sent = np.flip(np_sent, axis=0).copy()
tensor_sent = torch.LongTensor(np_sent)
if self.gpu:
tensor_sent = tensor_sent.cuda()
src_encoding, decoder_init_state = self.encoder[key].encode_one_sent(
tensor_sent)
if config.greedy_search:
tgt_tensor, score = self.decoder.greedy_search(src_encoding,
decoder_init_state,
max_step,
replace=replace)
tgt_np = tgt_tensor.cpu().detach().numpy()
tgt_sent = []
for i in tgt_np[1:-1]:
if i >= 0:
tgt_sent.append(self.vocab_tgt.id2word[i])
else:
tgt_sent.append(src_sent[-i])
hypotheses = [Hypothesis(tgt_sent, score)]
else:
l = self.decoder.beam_search(src_encoding,
decoder_init_state,
max_step,
replace=replace)
hypotheses = []
for tgt_tensor, score in l:
tgt_np = tgt_tensor.cpu().detach().numpy()
tgt_sent = []
for i in tgt_np[1:-1]:
if i > 0:
tgt_sent.append(self.vocab_tgt.id2word[i])
else:
tgt_sent.append(src_sent[-i])
hypotheses.append(Hypothesis(tgt_sent, score))
return hypotheses
def beam_search(self,
src_sent: List[str],
max_step=None,
replace=False,
**kwargs) -> List[Hypothesis]:
np_sent = np.array([self.vocab.src[word] for word in src_sent])
tensor_sent = torch.LongTensor(np_sent)
if self.gpu:
tensor_sent = tensor_sent.cuda()
src_encoding, decoder_init_state = self.encoder.encode_one_sent(
tensor_sent)
#print(src_sent, np_sent, src_encoding.size())
if dconfig.greedy_search:
tgt_tensor, score = self.decoder.greedy_search(src_encoding,
decoder_init_state,
max_step,
replace=replace)
tgt_np = tgt_tensor.cpu().detach().numpy()
tgt_sent = []
for i in tgt_np[1:-1]:
if i >= 0:
tgt_sent.append(self.vocab.tgt.id2word[i])
else:
tgt_sent.append(src_sent[-i])
hypotheses = [Hypothesis(tgt_sent, score)]
else:
l = self.decoder.beam_search(src_encoding,
decoder_init_state,
max_step,
replace=replace)
hypotheses = []
for tgt_tensor, score in l:
tgt_np = tgt_tensor.cpu().detach().numpy()
tgt_sent = []
for i in tgt_np[1:-1]:
if i > 0:
tgt_sent.append(self.vocab.tgt.id2word[i])
else:
tgt_sent.append(src_sent[-i])
hypotheses.append(Hypothesis(tgt_sent, score))
return hypotheses
def create_pairwise_hypotheses(self, detections, scene_start, scene_end):
# This simply creates pairwise hypotheses for all pairs
# of detections that are at most dt apart, and between
# which we can interpolate so that IoU of hypotheses
# is > 0. This is obviously an overcomplete set.
self.logger.info("Creating pairwise hypotheses...")
pairwise_hypothesis = []
if len(detections) == 0:
return pairwise_hypothesis
detections = sorted(detections, key=lambda detection: detection.time)
detections_by_time = [[detections[0]]]
for detection in detections[1:]:
if detection.time == detections_by_time[-1][0].time:
detections_by_time[-1].append(detection)
else:
detections_by_time.append([detection])
for idx_now, detections_now in enumerate(detections_by_time):
for detections_nxt in detections_by_time[idx_now + 1:]:
if len(detections_now) == 0 or len(detections_nxt) == 0:
continue
if detections_nxt[0].time - detections_now[0].time >\
self.pairwise_max_dt:
break
for detection_now in detections_now:
for detection_nxt in detections_nxt:
tracklet = interpolate_tracklet(
detection_now, detection_nxt)
bboxes = np.vstack(
[detection.bbox for detection in tracklet])
ious = IoU(bboxes[:-1], bboxes[1:])
if np.min(ious) >= self.pairwise_min_iou:
tracklet_fin = [
None for when in range(scene_start, scene_end)
]
for did, det in enumerate(tracklet):
tracklet_fin[det.time - scene_start] = det
if did > 0 and did < len(tracklet) - 1:
det.confidence = \
LabelStorage.instance.min_det_confidence
pairwise_hypothesis.append(
Hypothesis(tracklet=tracklet_fin))
self.logger.info("Done: total %d", len(pairwise_hypothesis))
return pairwise_hypothesis
def create_merged_hypotheses(self, hypotheses, active_pairwise, track_len):
# Merge hypotheses of current length with all possible pairwise hypotheses
# to create candidates of hypotheses of longer lengths.
# To do that, we consider hypotheses A----B, and pairwise hypotheses B--C,
# to create A---B--C.
self.logger.info("Creating merged hypotheses")
out = []
heads = [{} for _ in range(track_len)]
for cur_len in range(1, track_len):
for h in hypotheses[cur_len]:
ptr = -1
while h.tracklet[ptr] is None:
ptr -= 1
detection = h.tracklet[ptr]
if detection not in heads[cur_len].keys():
heads[cur_len][detection] = []
heads[cur_len][detection].append(h)
for h in active_pairwise:
ptr = 0
while h.tracklet[ptr] is None:
ptr += 1
detection = h.tracklet[ptr]
tail_ptr = len(h.tracklet) - 1
while h.tracklet[tail_ptr] is None:
tail_ptr -= 1
need_len = track_len - (tail_ptr - ptr + 1) + 1
if detection in heads[need_len].keys():
for head in heads[need_len][detection]:
new_tracklet = [det for det in head.tracklet]
for did, d in enumerate(h.tracklet):
if d is not None:
try:
new_tracklet[did] = d
except:
print("Here")
out.append(Hypothesis(new_tracklet))
self.logger.info("Done: total %d", len(out))
return out
def do(self, hypotheses, previous_batch=[]):
self.logger.info("Doing action")
good_h = []
for h in hypotheses:
new_tracklet = [d for d in h.tracklet if d is not None]
int_tracklet = [new_tracklet[0]]
for d in new_tracklet[1:]:
int_tracklet += interpolate_tracklet(int_tracklet[-1], d)[1:]
good_h.append(Hypothesis(int_tracklet, h.score))
good_h[-1].outputs = deepcopy(h.outputs)
if len(good_h) == 0:
return [], []
tracks = []
hypos = []
goodhs = []
scores = np.asarray([hypothesis.score for hypothesis in good_h])
prev_ctr = 0
while True:
# First find the best match to anything in the previous batch
# as these are the ones that must be there.
if prev_ctr < len(previous_batch):
new_scores = []
for hid, h in enumerate(good_h):
matched = True
for did, det in enumerate(h.tracklet):
if did >= len(previous_batch[prev_ctr]):
break
if det is None:
matched = False
break
if det != previous_batch[prev_ctr][did]:
matched = False
if matched:
val = h.score
for track, hypo in zip(tracks, hypos):
if hypotheses_IoU(hypo, hypotheses[hid]) >=\
self.iou_cutoff:
val = -1e9
new_scores.append(val)
else:
new_scores.append(-2e9)
best_idx = np.argmax(new_scores)
self.logger.debug("Matching previous batch with track %d"
" with score %0.3f",
best_idx, new_scores[best_idx])
prev_ctr += 1
else:
# Otherwise simply pick the one with the best score
best_idx = np.argmax(scores)
if scores[best_idx] <= self.score_cutoff:
break
self.logger.info("Selecting track %d with score %0.3f",
best_idx, scores[best_idx])
tracks.append(good_h[best_idx].tracklet)
# Now put score of anything which overlaps with the selected
# track to -infty
hypos.append(hypotheses[best_idx])
goodhs.append(good_h[best_idx])
for hid, hypothesis in enumerate(hypotheses):
if hid == best_idx or\
hypotheses_IoU(hypos[-1], hypothesis) >=\
self.iou_cutoff:
scores[hid] = -1e9
return tracks, hypos
def beam_search(self,
input_tensor,
input_lengths=None,
ext_vocab_size=None,
beam_size=4,
*,
min_out_len=1,
max_out_len=None,
len_in_words=True) -> List[Hypothesis]:
"""
:param input_tensor: tensor of word indices, (src seq len, batch size); for now, batch size has
to be 1
:param input_lengths: see explanation in `EncoderRNN`
:param ext_vocab_size: see explanation in `DecoderRNN`
:param beam_size: the beam size
:param min_out_len: required minimum output length
:param max_out_len: required maximum output length (if None, use the model's own value)
:param len_in_words: if True, count output length in words instead of tokens (i.e. do not count
punctuations)
:return: list of the best decoded sequences, in descending order of probability
Use beam search to generate summaries.
"""
batch_size = input_tensor.size(1)
assert batch_size == 1
if max_out_len is None:
max_out_len = self.max_dec_steps - 1 # max_out_len doesn't count EOS
# encode
encoder_hidden = self.encoder.init_hidden(batch_size)
# encoder_embedded: (input len, batch size, embed size)
encoder_embedded = self.embedding(
self.filter_oov(input_tensor, ext_vocab_size))
encoder_outputs, encoder_hidden = \
self.encoder(encoder_embedded, encoder_hidden, input_lengths)
if self.enc_dec_adapter is None:
decoder_hidden = encoder_hidden
else:
decoder_hidden = self.enc_dec_adapter(encoder_hidden)
# turn batch size from 1 to beam size (by repeating)
# if we want dynamic batch size, the following must be created for all possible batch sizes
encoder_outputs = encoder_outputs.expand(-1, beam_size,
-1).contiguous()
input_tensor = input_tensor.expand(-1, beam_size).contiguous()
# decode
hypos = [Hypothesis([self.vocab.SOS], [], decoder_hidden, [], [], 1)]
complete_results = []
step = 0
while hypos and step < 2 * max_out_len: # prevent infinitely generating punctuations
# make batch size equal to beam size (n_hypos <= beam size)
n_hypos = len(hypos)
if n_hypos < beam_size:
hypos.extend(hypos[-1] for _ in range(beam_size - n_hypos))
# assemble existing hypotheses into a batch
decoder_input = torch.tensor([h.tokens[-1] for h in hypos],
device=DEVICE)
decoder_hidden = torch.cat([h.dec_hidden for h in hypos], 1)
if self.dec_attn: # dim 0 is decoding step, dim 1 is beam batch
decoder_states = torch.cat(
[torch.cat(h.dec_states, 0) for h in hypos], 1)
else:
decoder_states = None
if self.enc_attn_cover:
enc_attn_weights = [
torch.cat([h.enc_attn_weights[i] for h in hypos], 1)
for i in range(step)
]
else:
enc_attn_weights = []
if enc_attn_weights:
coverage_vector = self.get_coverage_vector(
enc_attn_weights) # shape: (beam size, src len)
else:
coverage_vector = None
# run the decoder over the assembled batch
decoder_embedded = self.embedding(
self.filter_oov(decoder_input, ext_vocab_size))
decoder_output, decoder_hidden, dec_enc_attn, dec_prob_ptr = \
self.decoder(decoder_embedded, decoder_hidden, encoder_outputs,
decoder_states, coverage_vector,
encoder_word_idx=input_tensor, ext_vocab_size=ext_vocab_size)
top_v, top_i = decoder_output.data.topk(
beam_size) # shape of both: (beam size, beam size)
# create new hypotheses
new_hypos = []
for in_idx in range(n_hypos):
for out_idx in range(beam_size):
new_tok = top_i[in_idx][out_idx].item()
new_prob = top_v[in_idx][out_idx].item()
if len_in_words:
if new_tok < 3:
non_word = True
elif new_tok < self.vocab_size:
token_str = self.vocab[new_tok]
non_word = word_detector.search(
token_str) is None or token_str == '<P>'
else: # OOV is assumed to be word-only, not punctuation
non_word = False
else:
non_word = new_tok == self.vocab.EOS # only SOS & EOS don't count
new_hypo = hypos[in_idx].create_next(
new_tok, new_prob,
decoder_hidden[0][in_idx].unsqueeze(0).unsqueeze(0),
self.dec_attn,
dec_enc_attn[in_idx].unsqueeze(0).unsqueeze(0)
if dec_enc_attn is not None else None, non_word)
new_hypos.append(new_hypo)
# process the new hypotheses
new_hypos = sorted(new_hypos, key=lambda h: -h.avg_log_prob)
hypos = []
new_complete_results = []
for nh in new_hypos:
if nh.tokens[-1] == self.vocab.EOS: # a complete hypothesis
if len(new_complete_results
) < beam_size and min_out_len <= len(
nh) <= max_out_len:
new_complete_results.append(nh)
elif len(hypos) < beam_size and len(
nh) <= max_out_len: # an incomplete hypothesis
hypos.append(nh)
if len(hypos) >= beam_size and len(
new_complete_results) >= beam_size:
break # neither of the lists accept new items
complete_results.extend(new_complete_results)
step += 1
if complete_results:
return sorted(complete_results,
key=lambda h: -h.avg_log_prob)[:beam_size]
else:
return hypos
def beam_search(self,
input_tensor,
input_lengths=None,
ext_vocab_size=None,
beam_size=4,
*,
min_out_len=1,
max_out_len=None,
len_in_words=True,
mask=None) -> List[Hypothesis]:
"""
:param input_tensor: tensor of word indices, (src seq len, batch size); for now, batch size has
to be 1
:param input_lengths: see explanation in `EncoderRNN`
:param ext_vocab_size: see explanation in `DecoderRNN`
:param beam_size: the beam size
:param min_out_len: required minimum output length
:param max_out_len: required maximum output length (if None, use the model's own value)
:param len_in_words: if True, count output length in words instead of tokens (i.e. do not count
punctuations)
:return: list of the best decoded sequences, in descending order of probability
Use beam search to generate summaries.
"""
batch_size = input_tensor.size(1) # 1
assert batch_size == 1
if max_out_len is None:
max_out_len = self.max_dec_steps - 1 # max_out_len doesn't count EOS
# encode
encoder_hidden = self.encoder.init_hidden(batch_size)
# encoder_embedded: (input len, batch size, embed size)
encoder_embedded = self.embedding(
self.filter_oov(input_tensor, ext_vocab_size))
# output: [input_length, 1, hidden]
# hidden: [1, 1, hidden]
encoder_outputs, encoder_hidden = \
self.encoder(encoder_embedded, encoder_hidden, input_lengths)
if self.enc_dec_adapter is None:
decoder_hidden = encoder_hidden
else:
decoder_hidden = self.enc_dec_adapter(encoder_hidden)
# turn batch size from 1 to beam size (by repeating)
# if we want dynamic batch size, the following must be created for all possible batch sizes
# encoder_outputs:[src_len, beam_size, hidden_size] 相当于复制了四套encoder的状态
encoder_outputs = encoder_outputs.expand(-1, beam_size,
-1).contiguous()
# input_tensor：[src_len, beam_size] 将输入tensor复制四份
input_tensor = input_tensor.expand(-1, beam_size).contiguous()
# decode
# 一个Hypothesis是一个预测，最后返回beam size大小的Hypothesis列表
# 初始化一个hypos，其中tokens以SOS开始，表示句子开始，
hypos = [Hypothesis([self.vocab.SOS], [], decoder_hidden, [], [], 1)]
results, backup_results = [], []
step = 0
while hypos and step < 2 * max_out_len: # prevent infinitely generating punctuations（标点）
# make batch size equal to beam size (n_hypos <= beam size)
n_hypos = len(hypos) # 最开始是1
if n_hypos < beam_size:
# 最后结果是一个长度为beam size的列表list，应该只会在第一次循环执行这个，相当于初始化
hypos.extend(hypos[-1] for _ in range(beam_size - n_hypos))
# assemble existing hypotheses into a batch [1, beam_size]
# 取每一个hypo最后一个词
decoder_input = torch.tensor([h.tokens[-1] for h in hypos],
device=DEVICE)
# decoder_hidden：[1, beam_size, hidden_size]
decoder_hidden = torch.cat([h.dec_hidden for h in hypos], 1)
if self.dec_attn and step > 0: # dim 0 is decoding step, dim 1 is beam batch
decoder_states = torch.cat(
[torch.cat(h.dec_states, 0) for h in hypos], 1)
else:
decoder_states = None
if self.enc_attn_cover:
# 获得encoder attention
enc_attn_weights = [
torch.cat([h.enc_attn_weights[i] for h in hypos], 1)
for i in range(step)
]
else:
enc_attn_weights = []
if enc_attn_weights:
coverage_vector = self.get_coverage_vector(
enc_attn_weights) # shape: (beam size, src len)
else:
coverage_vector = None
# run the decoder over the assembled batch
decoder_embedded = self.embedding(
self.filter_oov(decoder_input, ext_vocab_size))
decoder_output, decoder_hidden, dec_enc_attn, dec_prob_ptr = \
self.decoder(decoder_embedded, decoder_hidden, encoder_outputs,
decoder_states, coverage_vector,
encoder_word_idx=input_tensor, ext_vocab_size=ext_vocab_size, mask=mask)
# (beam size, beam size) 相当于是 (batch_size, beam size)
# top_v : value
# top_i : index
top_v, top_i = decoder_output.data.topk(
beam_size) # shape of both: (beam size, beam size)
# create new hypotheses
new_hypos = []
for in_idx in range(n_hypos):
for out_idx in range(beam_size):
new_tok = top_i[in_idx][out_idx].item()
new_prob = top_v[in_idx][out_idx].item()
if len_in_words:
non_word = not self.vocab.is_word(new_tok)
else:
non_word = new_tok == self.vocab.EOS # only SOS & EOS don't count
new_hypo = hypos[in_idx].create_next(
new_tok, new_prob,
decoder_hidden[0][in_idx].unsqueeze(0).unsqueeze(0),
self.dec_attn,
dec_enc_attn[in_idx].unsqueeze(0).unsqueeze(0)
if dec_enc_attn is not None else None, non_word)
new_hypos.append(new_hypo)
# process the new hypotheses
# 降序排序
new_hypos = sorted(new_hypos, key=lambda h: -h.avg_log_prob)
hypos = []
new_complete_results, new_incomplete_results = [], []
for nh in new_hypos:
length = len(nh)
if nh.tokens[-1] == self.vocab.EOS: # a complete hypothesis
if len(
new_complete_results
) < beam_size and min_out_len <= length <= max_out_len:
new_complete_results.append(nh)
elif len(
hypos
) < beam_size and length < max_out_len: # an incomplete hypothesis
hypos.append(nh)
elif length == max_out_len and len(
new_incomplete_results) < beam_size:
new_incomplete_results.append(nh)
if new_complete_results:
results.extend(new_complete_results)
elif new_incomplete_results:
backup_results.extend(new_incomplete_results)
step += 1
if not results: # if no sequence ends with EOS within desired length, fallback to sequences
results = backup_results # that are "truncated" at the end to max_out_len
return sorted(results, key=lambda h: -h.avg_log_prob)[:beam_size]
def do(self,
detections,
model,
previous_batch,
scene_start,
scene_end,
temp=1e-6):
self.logger.info("Doing action")
_detections = [det for det in detections]
# Hypotheses containing only one detection
unitary = []
for d in _detections:
tracklet = [None for _ in range(scene_start, scene_end)]
tracklet[d.time - scene_start] = d
h = Hypothesis(tracklet=tracklet)
unitary.append(h)
model.score(unitary)
# During inference we can try to ignore hypothesis with very low scores
# to speed up the process.
if self.nms_option.endswith("ignore"):
ignore_cutoff = float(self.nms_option.split('-')[-2])
else:
ignore_cutoff = -1e9
# That includes ignoring very low quality detections.
detections = []
for did in range(len(unitary)):
if unitary[did].score * (scene_end - scene_start) < ignore_cutoff:
pass
else:
detections.append(_detections[did])
# Creating pairwise hypotheses by interpolating between pairs of detections.
pairwise_hypotheses = self.create_pairwise_hypotheses(
detections, scene_start, scene_end)
# Populating hypotheses array.
# Hypotheses[1] = unitary detections
# Hypotheses[l] = pairwise hypotheses between detections l-1 frames apart
if len(detections) > 0:
t_min = min([detection.time for detection in detections])
t_max = max([detection.time for detection in detections])
max_track_len = t_max - t_min + 1
else:
max_track_len = scene_end - scene_start + 1
hypotheses = [[] for _ in range(max_track_len + 1)]
for detection in detections:
tracklet = [None for when in range(scene_start, scene_end)]
tracklet[detection.time - scene_start] = detection
hypotheses[1].append(Hypothesis(tracklet))
for hypothesis in pairwise_hypotheses:
h_start = min(
[det.time for det in hypothesis.tracklet if det is not None])
h_end = max(
[det.time for det in hypothesis.tracklet if det is not None])
hypotheses[h_end - h_start + 1].append(hypothesis)
active_pairwise = []
all_hypos = []
for track_len in range(1, len(hypotheses)):
self.logger.info("Working with track_len %d", track_len)
# Compute scores for all hypotheses of the current length
model.score(hypotheses[track_len])
all_hypos += hypotheses[track_len]
# Run NMS - having at most 1 hypothesis of every length
# with every starting point.
hypotheses[track_len] = self.nms(hypotheses[track_len],
scene_start, scene_end, temp)
# After NMS, let's add the forced hypotheses from the prev batch:
# This is required to make sure that we can at least somehow link
# our current hypotheses to the ones in the previous batch.
push_force = []
for tracklet in previous_batch:
if len(tracklet) == track_len:
full_tracklet = [
None for when in range(scene_start, scene_end)
]
for detection in tracklet:
full_tracklet[detection.time - scene_start] = detection
push_force.append(Hypothesis(full_tracklet))
model.score(push_force)
hypotheses[track_len] += push_force
# Consider pairwise hypotheses which we will use to grow further.
# In particular, if we have two pairwise hypotheses A---B and B---C,
# they could be used to grow into A---B---C.
for hypothesis in hypotheses[track_len]:
sum_non_intp = 0
for detection in hypothesis.tracklet:
if detection is None: continue
if not hasattr(detection, "interpolated"):
sum_non_intp += 1
elif not detection.interpolated:
sum_non_intp += 1
if sum_non_intp == 2:
active_pairwise.append(hypothesis)
# Create additional candidates for longer hypotheses. We will NMS then
# when we process the hypotheses of that length in this loop.
if track_len < max_track_len:
merged = self.create_merged_hypotheses(hypotheses,
active_pairwise,
track_len + 1)
for hypothesis in merged:
hypotheses[track_len + 1].append(hypothesis)
return [
hypothesis for track_len in range(1, max_track_len + 1)
for hypothesis in hypotheses[track_len]
], all_hypos
def generate_beam(self, src1, src2):
src1_mat, src2_mat, src1_w1dt, src2_w1dt, decoder_state = self.encoder_forward(
src1, src2
)
hypothesis_list = [
Hypothesis(
text_list=[self.tgt_vocab.str2int(EOS)],
decoder_state=decoder_state,
c1_t=dy.vecInput(2 * HIDDEN_DIM),
c2_t=dy.vecInput(2 * HIDDEN_DIM),
prev_coverage=self.get_coverage(
a_t=dy.vecInput(len(src1)),
training=False,
prev_coverage=dy.vecInput(len(src1)),
),
score=0.0,
p_gens=[],
)
]
completed_list = []
for t in range(int(len(src1) * 1.1)):
new_hyp_list = []
new_hyp_scores = []
for hyp in hypothesis_list:
last_output_embeddings = self.tgt_lookup[hyp.text_list[-1]]
a_t, c1_t = self.attend(
src1_mat,
hyp.decoder_state,
src1_w1dt,
self.att1_w2,
self.att1_v,
hyp.prev_coverage,
)
if not self.single_source:
_, c2_t = self.attend(
src2_mat,
hyp.decoder_state,
src2_w1dt,
self.att2_w2,
self.att2_v,
None,
)
else:
c2_t = dy.vecInput(2 * HIDDEN_DIM)
x_t = dy.concatenate([c1_t, c2_t, last_output_embeddings])
decoder_state = hyp.decoder_state.add_input(x_t)
probs = dy.softmax(self.dec_w * decoder_state.output() + self.dec_b)
probs, cur_p_gen = self.get_pointergen_probs(
c1_t, decoder_state, x_t, a_t, probs, src1
)
probs = probs.npvalue()
for ind in range(len(probs)):
text_list = hyp.text_list + [ind]
p_gens = hyp.p_gens + [cur_p_gen]
score = (hyp.score + math.log(probs[ind])) / (len(text_list) ** 0.0)
coverage = self.get_coverage(a_t, hyp.prev_coverage, training=False)
new_hyp_list.append(
Hypothesis(
text_list=text_list,
decoder_state=decoder_state,
c1_t=c1_t,
c2_t=c2_t,
prev_coverage=coverage,
score=score,
p_gens=p_gens,
)
)
new_hyp_scores.append(score)
top_inds = np.argpartition(np.array(new_hyp_scores), -self.beam_size)[
-self.beam_size :
]
new_hyp_list = np.array(new_hyp_list)[top_inds]
hypothesis_list = []
for new_hyp in new_hyp_list:
if new_hyp.text_list[-1] == self.tgt_vocab.str2int(EOS) and t > 0:
completed_list.append(new_hyp)
else:
hypothesis_list.append(new_hyp)
if len(completed_list) >= self.beam_size:
break
if len(completed_list) == 0:
sorted(hypothesis_list, key=lambda x: x.score, reverse=True)
completed_list = [hypothesis_list[0]]
for hyp in completed_list:
hyp.text_list = [self.tgt_vocab.int2str(i) for i in hyp.text_list]
top_hyp = sorted(completed_list, key=lambda x: x.score, reverse=True)[0]
return "".join(top_hyp.text_list).replace(EOS, "").strip(), top_hyp.p_gens[1:-1]