def compute_batch_total_bleu(E_ref, E_cand, target_sos, target_eos):
'''Compute the total BLEU score over elements in a batch
Parameters
----------
E_ref : torch.LongTensor
A batch of reference transcripts of shape ``(T, N)``, including
start-of-sequence tags and right-padded with end-of-sequence tags.
E_cand : torch.LongTensor
A batch of candidate transcripts of shape ``(T', N)``, also including
start-of-sequence and end-of-sequence tags.
target_sos : int
The ID of the start-of-sequence tag in the target vocabulary.
target_eos : int
The ID of the end-of-sequence tag in the target vocabulary.
Returns
-------
total_bleu : float
The sum total BLEU score for across all elements in the batch. Use
n-gram precision 4.
'''
# you can use E_ref.tolist() to convert the LongTensor to a python list
# of numbers
total_bleu = 0
for ref, cand in zip(E_ref.transpose(0,1).tolist(), E_cand.transpose(0,1).tolist()):
ref = [word for word in ref if (word != target_eos and word != target_sos)]
cand = [word for word in cand if (word != target_eos and word != target_sos)]
total_bleu += a2_bleu_score.BLEU_score(ref, cand, 4)
return total_bleu
def compute_batch_total_bleu(E_ref, E_cand, target_sos, target_eos):
'''Compute the total BLEU score over elements in a batch
Parameters
----------
E_ref : torch.LongTensor
A batch of reference transcripts of shape ``(T, N)``, including
start-of-sequence tags and right-padded with end-of-sequence tags.
E_cand : torch.LongTensor
A batch of candidate transcripts of shape ``(T', N)``, also including
start-of-sequence and end-of-sequence tags.
target_sos : int
The ID of the start-of-sequence tag in the target vocabulary.
target_eos : int
The ID of the end-of-sequence tag in the target vocabulary.
Returns
-------
total_bleu : float
The sum total BLEU score for across all elements in the batch. Use
n-gram precision 4.
'''
# you can use E_ref.tolist() to convert the LongTensor to a python list
# of numbers
total = 0
eos = str(target_eos)
sos = str(target_sos)
E_ref = E_ref.permute(1, 0).tolist()
E_cand = E_cand.permute(1, 0).tolist()
for ref, cand in zip(E_ref, E_cand):
# make sure that we don't evaluate the eos and sos tokens.
ref = [str(x) for x in ref if str(x) != eos and str(x) != sos]
cand = [str(x) for x in cand if str(x) != eos and str(x) != sos]
total += a2_bleu_score.BLEU_score(ref, cand, 4)
return total
def compute_batch_total_bleu(E_ref, E_cand, target_sos, target_eos):
'''Compute the total BLEU score over elements in a batch
Parameters
----------
E_ref : torch.LongTensor
A batch of reference transcripts of shape ``(T, N)``, including
start-of-sequence tags and right-padded with end-of-sequence tags.
E_cand : torch.LongTensor
A batch of candidate transcripts of shape ``(T', N)``, also including
start-of-sequence and end-of-sequence tags.
target_sos : int
The ID of the start-of-sequence tag in the target vocabulary.
target_eos : int
The ID of the end-of-sequence tag in the target vocabulary.
Returns
-------
total_bleu : float
The sum total BLEU score for across all elements in the batch. Use
n-gram precision 4.
'''
# you can use E_ref.tolist() to convert the LongTensor to a python list
# of numbers
total_bleu = 0
reference = E_ref.tolist()
candidate = E_cand.tolist()
for i, j in zip(range(len(reference)), range(len(candidate))):
ref = reference[i]
cand = candidate[j]
str_ref = []
str_cand = []
str_target_eos = str(target_eos)
str_target_sos = str(target_sos)
for s in ref:
str_ref.append(str(s))
for s in cand:
str_cand.append(str(s))
#print("cand Length" + str(len(str_cand)))
#print(str_cand)
#print("ref Length" + str(len(str_ref)))
if str_target_sos in str_ref:
str_ref.remove(str_target_sos)
if str_target_eos in str_ref:
str_ref.remove(str_target_eos)
if str_target_sos in str_cand:
str_cand.remove(str_target_sos)
if str_target_eos in str_cand:
str_cand.remove(str_target_eos)
bleu_score = a2_bleu_score.BLEU_score(str_ref, str_cand, 4)
total_bleu += bleu_score
return total_bleu
def compute_batch_total_bleu(E_ref, E_cand, target_sos, target_eos):
'''Compute the total BLEU score over elements in a batch
Parameters
----------
E_ref : torch.LongTensor
A batch of reference transcripts of shape ``(T, N)``, including
start-of-sequence tags and right-padded with end-of-sequence tags.
E_cand : torch.LongTensor
A batch of candidate transcripts of shape ``(T', N)``, also including
start-of-sequence and end-of-sequence tags.
target_sos : int
The ID of the start-of-sequence tag in the target vocabulary.
target_eos : int
The ID of the end-of-sequence tag in the target vocabulary.
Returns
-------
total_bleu : float
The sum total BLEU score for across all elements in the batch. Use
n-gram precision 4.
'''
# you can use E_ref.tolist() to convert the LongTensor to a python list
# of numbers
total_bleu = 0
E_ref = E_ref.tolist()
# print(E_ref)
E_cand = E_cand.tolist()
# print(E_cand)
for i in range(len(E_ref[0])):
# print("Sequence ", i)
ref = [item[i] for item in E_ref]
# ref = E_ref[i]
# print("ref", ref)
cand = [item[i] for item in E_cand]
# cand = E_cand[i]
# print("cand", cand)
# ref = E_ref[i]
ref = [x for x in ref if x != target_sos]
ref = [x for x in ref if x != target_eos]
# cand = E_cand[i]
cand = [x for x in cand if x != target_sos]
cand = [x for x in cand if x != target_eos]
total_bleu += a2_bleu_score.BLEU_score(ref, cand, 4)
return total_bleu
def compute_batch_total_bleu(E_ref, E_cand, target_sos, target_eos):
'''Calculates the total BLEU score over elements in a batch
Parameters
----------
E_ref : torch.LongTensor
A batch of reference transcripts of shape ``(T, M)``, including
start-of-sequence tags and right-padded with end-of-sequence tags.
E_cand : torch.LongTensor
A batch of candidate transcripts of shape ``(T', M)``, also including
start-of-sequence and end-of-sequence tags.
target_sos : int
The id of the start-of-sequence tag in the target vocabulary.
target_eos : int
The id of the end-of-sequence tag in the target vocabulary.
Returns
-------
total_bleu : float
The sum total BLEU score for across all elements in the batch. Use
n-gram precision 4.
'''
# you can use E_ref.tolist() to convert the LongTensor to a python list
# of numbers
total = 0
n_precision = 4
references = E_ref.tolist()
candidates = E_cand.tolist()
_, num_iters = E_ref.shape
for i in range(num_iters):
reference = []
for ref in E_ref[:, i].tolist():
if str(ref) != str(target_eos) and str(ref) != str(target_sos):
reference.append(ref)
candidate = []
for cand in E_cand[:, i].tolist():
if str(cand) != str(target_eos) and str(cand) != str(target_sos):
candidate.append(cand)
total = total + a2_bleu_score.BLEU_score(reference, candidate,
n_precision)
return total
def compute_batch_total_bleu(E_ref, E_cand, target_sos, target_eos):
'''Compute the total BLEU score over elements in a batch
Parameters
----------
E_ref : torch.LongTensor
A batch of reference transcripts of shape ``(T, N)``, including
start-of-sequence tags and right-padded with end-of-sequence tags.
E_cand : torch.LongTensor
A batch of candidate transcripts of shape ``(T', N)``, also including
start-of-sequence and end-of-sequence tags.
target_sos : int
The ID of the start-of-sequence tag in the target vocabulary.
target_eos : int
The ID of the end-of-sequence tag in the target vocabulary.
Returns
-------
total_bleu : float
The sum total BLEU score for across all elements in the batch. Use
n-gram precision 4.
'''
total_bleu = 0
N = E_ref.shape[1]
for i in range(N):
cand = E_cand[:, i].tolist()
ref = E_ref[:, i].tolist()
while target_eos in cand:
_ = cand.pop(cand.index(target_eos))
while target_sos in cand:
_ = cand.pop(cand.index(target_sos))
while target_eos in ref:
_ = ref.pop(ref.index(target_eos))
while target_sos in ref:
_ = ref.pop(ref.index(target_sos))
total_bleu += a2_bleu_score.BLEU_score(ref, cand, 4)
return (total_bleu)
def compute_batch_total_bleu(E_ref, E_cand, target_sos, target_eos):
'''Calculates the total BLEU score over elements in a batch
Parameters
----------
E_ref : torch.LongTensor
A batch of reference transcripts of shape ``(T, M)``, including
start-of-sequence tags and right-padded with end-of-sequence tags.
E_cand : torch.LongTensor
A batch of candidate transcripts of shape ``(T', M)``, also including
start-of-sequence and end-of-sequence tags.
target_sos : int
The id of the start-of-sequence tag in the target vocabulary.
target_eos : int
The id of the end-of-sequence tag in the target vocabulary.
Returns
-------
total_bleu : float
The sum total BLEU score for across all elements in the batch. Use
n-gram precision 4.
'''
# you can use E_ref.tolist() to convert the LongTensor to a python list
# of numbers
result = 0
E_ref = E_ref.permute(1, 0)
E_cand = E_cand.permute(1, 0)
for ref, cand in zip(E_ref.tolist(), E_cand.tolist()):
temp_ref = []
for item in ref:
if str(item) != str(target_eos) and str(item) != str(target_sos):
temp_ref.append(str(item))
temp_cand = []
for item in cand:
if str(item) != str(target_eos) and str(item) != str(target_sos):
temp_cand.append(str(item))
result += a2_bleu_score.BLEU_score(temp_ref, temp_cand, 4)
return result
def test_bleu(ids):
reference = '''\
it is a guide to action that ensures that the military will always heed
party commands'''.strip().split()
candidate = '''\
it is a guide to action which ensures that the military always obeys the
commands of the party'''.strip().split()
if ids:
# should work with token ids (ints) as well
reference = [hash(word) for word in reference]
candidate = [hash(word) for word in candidate]
assert np.isclose(
a2_bleu_score.n_gram_precision(reference, candidate, 1),
15 / 18,
)
assert np.isclose(
a2_bleu_score.n_gram_precision(reference, candidate, 2),
8 / 17,
)
assert np.isclose(a2_bleu_score.brevity_penalty(reference, candidate), 1.)
assert np.isclose(a2_bleu_score.BLEU_score(reference, candidate, 2),
1 * ((15 * 8) / (18 * 17))**(1 / 2))
def compute_batch_total_bleu(E_ref, E_cand, target_sos, target_eos):
'''Compute the total BLEU score over elements in a batch
Parameters
----------
E_ref : torch.LongTensor
A batch of reference transcripts of shape ``(T, N)``, including
start-of-sequence tags and right-padded with end-of-sequence tags.
E_cand : torch.LongTensor
A batch of candidate transcripts of shape ``(T', N)``, also including
start-of-sequence and end-of-sequence tags.
target_sos : int
The ID of the start-of-sequence tag in the target vocabulary.
target_eos : int
The ID of the end-of-sequence tag in the target vocabulary.
Returns
-------
total_bleu : float
The sum total BLEU score for across all elements in the batch. Use
n-gram precision 4.
'''
# you can use E_ref.tolist() to convert the LongTensor to a python list
# of numbers
sum = 0
N = E_ref.shape[1]
for i in range(N):
reference_list = [
x for x in E_ref[:, i].tolist()
if x != target_sos and x != target_eos
]
candidate_list = [
x for x in E_cand[:, i].tolist()
if x != target_sos and x != target_eos
]
sum += a2_bleu_score.BLEU_score(reference_list, candidate_list, 4)
return sum
