def compute_batch_total_bleu(E_ref, E_cand, target_sos, target_eos): '''Compute the total BLEU score over elements in a batch Parameters ---------- E_ref : torch.LongTensor A batch of reference transcripts of shape ``(T, N)``, including start-of-sequence tags and right-padded with end-of-sequence tags. E_cand : torch.LongTensor A batch of candidate transcripts of shape ``(T', N)``, also including start-of-sequence and end-of-sequence tags. target_sos : int The ID of the start-of-sequence tag in the target vocabulary. target_eos : int The ID of the end-of-sequence tag in the target vocabulary. Returns ------- total_bleu : float The sum total BLEU score for across all elements in the batch. Use n-gram precision 4. ''' # you can use E_ref.tolist() to convert the LongTensor to a python list # of numbers total_bleu = 0 for ref, cand in zip(E_ref.transpose(0,1).tolist(), E_cand.transpose(0,1).tolist()): ref = [word for word in ref if (word != target_eos and word != target_sos)] cand = [word for word in cand if (word != target_eos and word != target_sos)] total_bleu += a2_bleu_score.BLEU_score(ref, cand, 4) return total_bleu
def compute_batch_total_bleu(E_ref, E_cand, target_sos, target_eos): '''Compute the total BLEU score over elements in a batch Parameters ---------- E_ref : torch.LongTensor A batch of reference transcripts of shape ``(T, N)``, including start-of-sequence tags and right-padded with end-of-sequence tags. E_cand : torch.LongTensor A batch of candidate transcripts of shape ``(T', N)``, also including start-of-sequence and end-of-sequence tags. target_sos : int The ID of the start-of-sequence tag in the target vocabulary. target_eos : int The ID of the end-of-sequence tag in the target vocabulary. Returns ------- total_bleu : float The sum total BLEU score for across all elements in the batch. Use n-gram precision 4. ''' # you can use E_ref.tolist() to convert the LongTensor to a python list # of numbers total = 0 eos = str(target_eos) sos = str(target_sos) E_ref = E_ref.permute(1, 0).tolist() E_cand = E_cand.permute(1, 0).tolist() for ref, cand in zip(E_ref, E_cand): # make sure that we don't evaluate the eos and sos tokens. ref = [str(x) for x in ref if str(x) != eos and str(x) != sos] cand = [str(x) for x in cand if str(x) != eos and str(x) != sos] total += a2_bleu_score.BLEU_score(ref, cand, 4) return total
def compute_batch_total_bleu(E_ref, E_cand, target_sos, target_eos): '''Compute the total BLEU score over elements in a batch Parameters ---------- E_ref : torch.LongTensor A batch of reference transcripts of shape ``(T, N)``, including start-of-sequence tags and right-padded with end-of-sequence tags. E_cand : torch.LongTensor A batch of candidate transcripts of shape ``(T', N)``, also including start-of-sequence and end-of-sequence tags. target_sos : int The ID of the start-of-sequence tag in the target vocabulary. target_eos : int The ID of the end-of-sequence tag in the target vocabulary. Returns ------- total_bleu : float The sum total BLEU score for across all elements in the batch. Use n-gram precision 4. ''' # you can use E_ref.tolist() to convert the LongTensor to a python list # of numbers total_bleu = 0 reference = E_ref.tolist() candidate = E_cand.tolist() for i, j in zip(range(len(reference)), range(len(candidate))): ref = reference[i] cand = candidate[j] str_ref = [] str_cand = [] str_target_eos = str(target_eos) str_target_sos = str(target_sos) for s in ref: str_ref.append(str(s)) for s in cand: str_cand.append(str(s)) #print("cand Length" + str(len(str_cand))) #print(str_cand) #print("ref Length" + str(len(str_ref))) if str_target_sos in str_ref: str_ref.remove(str_target_sos) if str_target_eos in str_ref: str_ref.remove(str_target_eos) if str_target_sos in str_cand: str_cand.remove(str_target_sos) if str_target_eos in str_cand: str_cand.remove(str_target_eos) bleu_score = a2_bleu_score.BLEU_score(str_ref, str_cand, 4) total_bleu += bleu_score return total_bleu
def compute_batch_total_bleu(E_ref, E_cand, target_sos, target_eos): '''Compute the total BLEU score over elements in a batch Parameters ---------- E_ref : torch.LongTensor A batch of reference transcripts of shape ``(T, N)``, including start-of-sequence tags and right-padded with end-of-sequence tags. E_cand : torch.LongTensor A batch of candidate transcripts of shape ``(T', N)``, also including start-of-sequence and end-of-sequence tags. target_sos : int The ID of the start-of-sequence tag in the target vocabulary. target_eos : int The ID of the end-of-sequence tag in the target vocabulary. Returns ------- total_bleu : float The sum total BLEU score for across all elements in the batch. Use n-gram precision 4. ''' # you can use E_ref.tolist() to convert the LongTensor to a python list # of numbers total_bleu = 0 E_ref = E_ref.tolist() # print(E_ref) E_cand = E_cand.tolist() # print(E_cand) for i in range(len(E_ref[0])): # print("Sequence ", i) ref = [item[i] for item in E_ref] # ref = E_ref[i] # print("ref", ref) cand = [item[i] for item in E_cand] # cand = E_cand[i] # print("cand", cand) # ref = E_ref[i] ref = [x for x in ref if x != target_sos] ref = [x for x in ref if x != target_eos] # cand = E_cand[i] cand = [x for x in cand if x != target_sos] cand = [x for x in cand if x != target_eos] total_bleu += a2_bleu_score.BLEU_score(ref, cand, 4) return total_bleu
def compute_batch_total_bleu(E_ref, E_cand, target_sos, target_eos): '''Calculates the total BLEU score over elements in a batch Parameters ---------- E_ref : torch.LongTensor A batch of reference transcripts of shape ``(T, M)``, including start-of-sequence tags and right-padded with end-of-sequence tags. E_cand : torch.LongTensor A batch of candidate transcripts of shape ``(T', M)``, also including start-of-sequence and end-of-sequence tags. target_sos : int The id of the start-of-sequence tag in the target vocabulary. target_eos : int The id of the end-of-sequence tag in the target vocabulary. Returns ------- total_bleu : float The sum total BLEU score for across all elements in the batch. Use n-gram precision 4. ''' # you can use E_ref.tolist() to convert the LongTensor to a python list # of numbers total = 0 n_precision = 4 references = E_ref.tolist() candidates = E_cand.tolist() _, num_iters = E_ref.shape for i in range(num_iters): reference = [] for ref in E_ref[:, i].tolist(): if str(ref) != str(target_eos) and str(ref) != str(target_sos): reference.append(ref) candidate = [] for cand in E_cand[:, i].tolist(): if str(cand) != str(target_eos) and str(cand) != str(target_sos): candidate.append(cand) total = total + a2_bleu_score.BLEU_score(reference, candidate, n_precision) return total
def compute_batch_total_bleu(E_ref, E_cand, target_sos, target_eos): '''Compute the total BLEU score over elements in a batch Parameters ---------- E_ref : torch.LongTensor A batch of reference transcripts of shape ``(T, N)``, including start-of-sequence tags and right-padded with end-of-sequence tags. E_cand : torch.LongTensor A batch of candidate transcripts of shape ``(T', N)``, also including start-of-sequence and end-of-sequence tags. target_sos : int The ID of the start-of-sequence tag in the target vocabulary. target_eos : int The ID of the end-of-sequence tag in the target vocabulary. Returns ------- total_bleu : float The sum total BLEU score for across all elements in the batch. Use n-gram precision 4. ''' total_bleu = 0 N = E_ref.shape[1] for i in range(N): cand = E_cand[:, i].tolist() ref = E_ref[:, i].tolist() while target_eos in cand: _ = cand.pop(cand.index(target_eos)) while target_sos in cand: _ = cand.pop(cand.index(target_sos)) while target_eos in ref: _ = ref.pop(ref.index(target_eos)) while target_sos in ref: _ = ref.pop(ref.index(target_sos)) total_bleu += a2_bleu_score.BLEU_score(ref, cand, 4) return (total_bleu)
def compute_batch_total_bleu(E_ref, E_cand, target_sos, target_eos): '''Calculates the total BLEU score over elements in a batch Parameters ---------- E_ref : torch.LongTensor A batch of reference transcripts of shape ``(T, M)``, including start-of-sequence tags and right-padded with end-of-sequence tags. E_cand : torch.LongTensor A batch of candidate transcripts of shape ``(T', M)``, also including start-of-sequence and end-of-sequence tags. target_sos : int The id of the start-of-sequence tag in the target vocabulary. target_eos : int The id of the end-of-sequence tag in the target vocabulary. Returns ------- total_bleu : float The sum total BLEU score for across all elements in the batch. Use n-gram precision 4. ''' # you can use E_ref.tolist() to convert the LongTensor to a python list # of numbers result = 0 E_ref = E_ref.permute(1, 0) E_cand = E_cand.permute(1, 0) for ref, cand in zip(E_ref.tolist(), E_cand.tolist()): temp_ref = [] for item in ref: if str(item) != str(target_eos) and str(item) != str(target_sos): temp_ref.append(str(item)) temp_cand = [] for item in cand: if str(item) != str(target_eos) and str(item) != str(target_sos): temp_cand.append(str(item)) result += a2_bleu_score.BLEU_score(temp_ref, temp_cand, 4) return result
def test_bleu(ids): reference = '''\ it is a guide to action that ensures that the military will always heed party commands'''.strip().split() candidate = '''\ it is a guide to action which ensures that the military always obeys the commands of the party'''.strip().split() if ids: # should work with token ids (ints) as well reference = [hash(word) for word in reference] candidate = [hash(word) for word in candidate] assert np.isclose( a2_bleu_score.n_gram_precision(reference, candidate, 1), 15 / 18, ) assert np.isclose( a2_bleu_score.n_gram_precision(reference, candidate, 2), 8 / 17, ) assert np.isclose(a2_bleu_score.brevity_penalty(reference, candidate), 1.) assert np.isclose(a2_bleu_score.BLEU_score(reference, candidate, 2), 1 * ((15 * 8) / (18 * 17))**(1 / 2))
def compute_batch_total_bleu(E_ref, E_cand, target_sos, target_eos): '''Compute the total BLEU score over elements in a batch Parameters ---------- E_ref : torch.LongTensor A batch of reference transcripts of shape ``(T, N)``, including start-of-sequence tags and right-padded with end-of-sequence tags. E_cand : torch.LongTensor A batch of candidate transcripts of shape ``(T', N)``, also including start-of-sequence and end-of-sequence tags. target_sos : int The ID of the start-of-sequence tag in the target vocabulary. target_eos : int The ID of the end-of-sequence tag in the target vocabulary. Returns ------- total_bleu : float The sum total BLEU score for across all elements in the batch. Use n-gram precision 4. ''' # you can use E_ref.tolist() to convert the LongTensor to a python list # of numbers sum = 0 N = E_ref.shape[1] for i in range(N): reference_list = [ x for x in E_ref[:, i].tolist() if x != target_sos and x != target_eos ] candidate_list = [ x for x in E_cand[:, i].tolist() if x != target_sos and x != target_eos ] sum += a2_bleu_score.BLEU_score(reference_list, candidate_list, 4) return sum