def _read_json(self, input_file): """Read a DRCD json file into a list of CRCDExample.""" def _is_chinese_char(cp): if ((cp >= 0x4E00 and cp <= 0x9FFF) or (cp >= 0x3400 and cp <= 0x4DBF) or (cp >= 0x20000 and cp <= 0x2A6DF) or (cp >= 0x2A700 and cp <= 0x2B73F) or (cp >= 0x2B740 and cp <= 0x2B81F) or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) or (cp >= 0x2F800 and cp <= 0x2FA1F)): return True return False def _is_punctuation(c): if c in [ '。', ',', '!', '?', ';', '、', ':', '(', ')', '-', '~', '「', '《', '》', ',', '」', '"', '“', '”', '$', '『', '』', '—', ';', '。', '(', ')', '-', '~', '。', '‘', '’', '─', ':' ]: return True return False def _tokenize_chinese_chars(text): """Because Chinese (and Japanese Kanji and Korean Hanja) does not have whitespace characters, we add spaces around every character in the CJK Unicode range before applying WordPiece. This means that Chinese is effectively character-tokenized. Note that the CJK Unicode block only includes Chinese-origin characters and does not include Hangul Korean or Katakana/Hiragana Japanese, which are tokenized with whitespace+WordPiece like all other languages.""" output = [] for char in text: cp = ord(char) if _is_chinese_char(cp) or _is_punctuation(char): if len(output) > 0 and output[-1] != SPIECE_UNDERLINE: output.append(SPIECE_UNDERLINE) output.append(char) output.append(SPIECE_UNDERLINE) else: output.append(char) return "".join(output) def is_whitespace(c): if c == " " or c == "\t" or c == "\r" or c == "\n" or ord( c) == 0x202F or ord(c) == 0x3000 or c == SPIECE_UNDERLINE: return True return False examples = [] with open(input_file, "r") as reader: input_data = json.load(reader)["data"] for entry in input_data: for paragraph in entry["paragraphs"]: paragraph_text = paragraph["context"] context = _tokenize_chinese_chars(paragraph_text) doc_tokens = [] char_to_word_offset = [] prev_is_whitespace = True for c in context: if is_whitespace(c): prev_is_whitespace = True else: if prev_is_whitespace: doc_tokens.append(c) else: doc_tokens[-1] += c prev_is_whitespace = False if c != SPIECE_UNDERLINE: char_to_word_offset.append(len(doc_tokens) - 1) for qa in paragraph["qas"]: qas_id = qa["id"] question_text = qa["question"] # Only select the first answer answer = qa["answers"][0] orig_answer_text = answer["text"] answer_offset = answer["answer_start"] while paragraph_text[answer_offset] in [ " ", "\t", "\r", "\n", "。", ",", ":", ":", ".", "," ]: answer_offset += 1 start_position = char_to_word_offset[answer_offset] answer_length = len(orig_answer_text) end_position = char_to_word_offset[answer_offset + answer_length - 1] # Only add answers where the text can be exactly recovered from the # document. If this CAN'T happen it's likely due to weird Unicode # stuff so we will just skip the example. # # Note that this means for training mode, every example is NOT # guaranteed to be preserved. actual_text = "".join( doc_tokens[start_position:(end_position + 1)]) cleaned_answer_text = "".join( tokenization.whitespace_tokenize(orig_answer_text)) if actual_text.find(cleaned_answer_text) == -1: logger.warning((actual_text, " vs ", cleaned_answer_text, " in ", qa)) continue example = DRCDExample(qas_id=qas_id, question_text=question_text, doc_tokens=doc_tokens, orig_answer_text=orig_answer_text, start_position=start_position, end_position=end_position) examples.append(example) return examples
def _read_json(self, input_file, if_has_answer, version_2_with_negative=False): """Read a SQuAD json file into a list of SquadExample.""" with open(input_file, "r") as reader: input_data = json.load(reader)["data"] def is_whitespace(c): if c == " " or c == "\t" or c == "\r" or c == "\n" or ord( c) == 0x202F: return True return False examples = [] for entry in input_data: for paragraph in entry["paragraphs"]: paragraph_text = paragraph["context"] doc_tokens = [] char_to_word_offset = [] prev_is_whitespace = True for c in paragraph_text: if is_whitespace(c): prev_is_whitespace = True else: if prev_is_whitespace: doc_tokens.append(c) else: doc_tokens[-1] += c prev_is_whitespace = False char_to_word_offset.append(len(doc_tokens) - 1) for qa in paragraph["qas"]: qas_id = qa["id"] question_text = qa["question"] start_position = None end_position = None orig_answer_text = None is_impossible = False if if_has_answer: if version_2_with_negative: is_impossible = qa["is_impossible"] # if (len(qa["answers"]) != 1) and (not is_impossible): # raise ValueError( # "For training, each question should have exactly 1 answer." # ) if not is_impossible: answer = qa["answers"][0] orig_answer_text = answer["text"] answer_offset = answer["answer_start"] answer_length = len(orig_answer_text) start_position = char_to_word_offset[answer_offset] end_position = char_to_word_offset[ answer_offset + answer_length - 1] # Only add answers where the text can be exactly recovered from the # document. If this CAN'T happen it's likely due to weird Unicode # stuff so we will just skip the example. # # Note that this means for training mode, every example is NOT # guaranteed to be preserved. actual_text = " ".join( doc_tokens[start_position:(end_position + 1)]) cleaned_answer_text = " ".join( tokenization.whitespace_tokenize( orig_answer_text)) if actual_text.find(cleaned_answer_text) == -1: logger.warning( "Could not find answer: '%s' vs. '%s'" % (actual_text, cleaned_answer_text)) continue else: start_position = -1 end_position = -1 orig_answer_text = "" example = SquadExample( qas_id=qas_id, question_text=question_text, doc_tokens=doc_tokens, orig_answer_text=orig_answer_text, start_position=start_position, end_position=end_position, is_impossible=is_impossible) examples.append(example) return examples
def _read_file(self, input_file, phase=False): """ 读入json格式数据集 """ def _is_chinese_char(cp): if ((cp >= 0x4E00 and cp <= 0x9FFF) or (cp >= 0x3400 and cp <= 0x4DBF) or (cp >= 0x20000 and cp <= 0x2A6DF) or (cp >= 0x2A700 and cp <= 0x2B73F) or (cp >= 0x2B740 and cp <= 0x2B81F) or (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) or (cp >= 0x2F800 and cp <= 0x2FA1F)): return True return False def _is_punctuation(c): if c in [ '。', ',', '!', '?', ';', '、', ':', '(', ')', '-', '~', '「', '《', '》', ',', '」', '"', '“', '”', '$', '『', '』', '—', ';', '。', '(', ')', '-', '~', '。', '‘', '’', '─', ':' ]: return True return False def _tokenize_chinese_chars(text): """ 中文汉字切分 """ output = [] for char in text: cp = ord(char) if _is_chinese_char(cp) or _is_punctuation(char): if len(output) > 0 and output[-1] != SPIECE_UNDERLINE: output.append(SPIECE_UNDERLINE) output.append(char) output.append(SPIECE_UNDERLINE) else: output.append(char) return "".join(output) def is_whitespace(c): if c == " " or c == "\t" or c == "\r" or c == "\n" or ord( c) == 0x202F or ord(c) == 0x3000 or c == SPIECE_UNDERLINE: return True return False examples = [] drop = 0 with open(input_file, "r") as reader: input_data = json.load(reader)["data"] for entry in input_data: for paragraph in entry["paragraphs"]: paragraph_text = paragraph["context"] context = _tokenize_chinese_chars(paragraph_text) doc_tokens = [] char_to_word_offset = [] prev_is_whitespace = True for c in context: if is_whitespace(c): prev_is_whitespace = True else: if prev_is_whitespace: doc_tokens.append(c) else: doc_tokens[-1] += c prev_is_whitespace = False if c != SPIECE_UNDERLINE: char_to_word_offset.append(len(doc_tokens) - 1) for qa in paragraph["qas"]: qas_id = qa["id"] question_text = qa["question"] if phase == 'predict': # 测试集部分没有答案 orig_answer_text = "" start_position = -1 end_position = -1 else: # 训练集/验证集选择第一个答案作为ground truth answer = qa["answers"][0] orig_answer_text = answer["text"] answer_offset = answer["answer_start"] while paragraph_text[answer_offset] in [ " ", "\t", "\r", "\n", "。", ",", ":", ":", ".", "," ]: answer_offset += 1 start_position = char_to_word_offset[answer_offset] answer_length = len(orig_answer_text) end_offset = answer_offset + answer_length - 1 if end_offset >= len(char_to_word_offset): end_offset = len(char_to_word_offset) - 1 end_position = char_to_word_offset[end_offset] if phase == "train": actual_text = "".join( doc_tokens[start_position:(end_position + 1)]) cleaned_answer_text = "".join( tokenization.whitespace_tokenize(orig_answer_text)) if actual_text.find(cleaned_answer_text) == -1: drop += 1 logger.warning((actual_text, " vs ", cleaned_answer_text, " in ", qa)) continue example = CMRC2018Example( qas_id=qas_id, question_text=question_text, doc_tokens=doc_tokens, orig_answer_text=orig_answer_text, start_position=start_position, end_position=end_position) examples.append(example) logger.warning("%i bad examples has been dropped" % drop) print(examples[1]) return examples