def _get_tokens(s):
"""get normalized tokens for both Chinese and English"""
if not s:
return []
# separate answers to en and ch pieces
ch_seq = ""
en_seq = ""
pos = 0
# Normalize and connect
final_tokens = []
while pos < len(s):
if check_chinese_char(s[pos]):
if en_seq != "":
final_tokens.extend(normalize_answer(en_seq).split())
en_seq = ""
ch_seq += s[pos]
else:
if ch_seq != "":
final_tokens.extend(normalize_chinese_answer(ch_seq))
ch_seq = ""
en_seq += s[pos]
pos += 1
if en_seq != "":
final_tokens.extend(normalize_answer(en_seq).split())
if ch_seq != "":
final_tokens.extend(normalize_chinese_answer(ch_seq))
return final_tokens
def _get_tokens(s):
"""get normalized tokens"""
if not s:
return []
return normalize_answer(s).split()
def exact_match_score(prediction, ground_truth):
"""computes exact match between prediction and ground truth"""
return int(normalize_answer(prediction) == normalize_answer(ground_truth))
