def map_fn_senti(examples, tokenizer): log.debug('load data %d' % len(examples)) contexts = [example['context'] for example in examples] tokenized_examples = tokenizer(contexts, max_seq_len=args.max_seq_len) tokenized_examples = convert_tokenizer_res_to_old_version( tokenized_examples) return tokenized_examples
def map_fn_DuCheckList(examples, args, tokenizer): # Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results # in one example possible giving several features when a context is long, each of those features having a # context that overlaps a bit the context of the previous feature. #NOTE: Almost the same functionality as HuggingFace's prepare_train_features function. The main difference is # that HugggingFace uses ArrowTable as basic data structure, while we use list of dictionary instead. if args.language == 'en': contexts = [ examples[i]['context'].encode('ascii', errors='replace').decode('UTF-8') for i in range(len(examples)) ] questions = [ examples[i]['question'].encode('ascii', errors='replace').decode('UTF-8') for i in range(len(examples)) ] else: contexts = [examples[i]['context'] for i in range(len(examples))] questions = [examples[i]['question'] for i in range(len(examples))] tokenized_examples = tokenizer(questions, contexts, stride=args.doc_stride, max_seq_len=args.max_seq_len) tokenized_examples = convert_tokenizer_res_to_old_version( tokenized_examples) # For validation, there is no need to compute start and end positions for i, tokenized_example in enumerate(tokenized_examples): # Grab the sequence corresponding to that example (to know what is the context and what is the question). sequence_ids = tokenized_example['token_type_ids'] # One example can give several spans, this is the index of the example containing this span of text. sample_index = tokenized_example['overflow_to_sample'] tokenized_examples[i]["example_id"] = examples[sample_index]['id'] # Set to None the offset_mapping that are not part of the context so it's easy to determine if a token # position is part of the context or not. if args.language == 'ch': tokenized_examples[i]["offset_mapping"] = [ (o if sequence_ids[k] == 1 else None) for k, o in enumerate(tokenized_example["offset_mapping"]) ] else: n = tokenized_example['offset_mapping'].index( (0, 0), 1) + 2 # context start position m = len(tokenized_example['offset_mapping'] ) - 1 # context end position + 1 tokenized_examples[i]["offset_mapping"] = [ (o if n <= k <= m else None) for k, o in enumerate(tokenized_example["offset_mapping"]) ] return tokenized_examples
def map_fn_DuCheckList(examples, args, tokenizer): # Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results # in one example possible giving several features when a context is long, each of those features having a # context that overlaps a bit the context of the previous feature. questions = [examples[i]['question'] for i in range(len(examples))] contexts = [ examples[i]['context'] + examples[i]['title'] for i in range(len(examples)) ] tokenized_examples = tokenizer(questions, contexts, stride=args.doc_stride, max_seq_len=args.max_seq_len) tokenized_examples = convert_tokenizer_res_to_old_version( tokenized_examples) log.debug('\nexample: %d' % len(examples)) log.debug('feature: %d\n' % len(tokenized_examples)) # For validation, there is no need to compute start and end positions for i, tokenized_example in enumerate(tokenized_examples): # Grab the sequence corresponding to that example (to know what is the context and what is the question). sequence_ids = tokenized_example['token_type_ids'] # One example can give several spans, this is the index of the example containing this span of text. sample_index = tokenized_example['overflow_to_sample'] tokenized_examples[i]["example_id"] = examples[sample_index]['id'] # Set to None the offset_mapping that are not part of the context so it's easy to determine if a token # position is part of the context or not. if args.language == 'ch': tokenized_examples[i]["offset_mapping"] = [ (o if sequence_ids[k] == 1 else None) for k, o in enumerate(tokenized_example["offset_mapping"]) ] else: n = tokenized_example['offset_mapping'].index( (0, 0), 1) + 2 # context start position m = len(tokenized_example['offset_mapping'] ) - 1 # context end position + 1 tokenized_examples[i]["offset_mapping"] = [ (o if n <= k < m else None) for k, o in enumerate(tokenized_example["offset_mapping"]) ] tokenized_examples[i]['question'] = examples[sample_index]['question'] tokenized_examples[i]['context'] = examples[sample_index]['context'] #tokenized_examples[i]['answer'] = examples[sample_index]['answers'][0] #tokenized_examples[i]['is_impossible'] = examples[sample_index]['is_impossible'] return tokenized_examples
def map_fn_senti(examples, tokenizer, language): print('load data %d' % len(examples)) if language == 'ch': q_name = "query" t_name = "title" else: q_name = "sentence1" t_name = "sentence2" queries = [example[q_name] for example in examples] titles = [example[t_name] for example in examples] tokenized_examples = tokenizer( queries, titles, max_seq_len=args.max_seq_len) tokenized_examples = convert_tokenizer_res_to_old_version( tokenized_examples) return tokenized_examples
def map_fn_senti(examples, tokenizer): print('load data %d' % len(examples)) if args.language == 'ch': query = 'query' title = 'title' else: query = 'sentence1' title = 'sentence2' queries = [example[query] for example in examples] titles = [example[title] for example in examples] tokenized_examples = tokenizer(queries, titles, max_seq_len=args.max_seq_len) tokenized_examples = convert_tokenizer_res_to_old_version( tokenized_examples) return tokenized_examples
def map_fn_senti(examples, tokenizer, args): log.debug('load data %d' % len(examples)) if args.language == 'en': contexts = [ example['context'].encode('ascii', errors='replace').decode('UTF-8') for example in examples ] else: contexts = [example['context'] for example in examples] tokenized_examples = tokenizer(contexts, max_seq_len=args.max_seq_len) tokenized_examples = convert_tokenizer_res_to_old_version( tokenized_examples) for i in range(len(tokenized_examples)): tokenized_examples[i]['offset_mapping'] = [ (0, 0) ] + tokenizer.get_offset_mapping( contexts[i])[:args.max_seq_len - 2] + [(0, 0)] return tokenized_examples
def map_fn_DuCheckList(examples, args, tokenizer): # Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results # in one example possible giving several features when a context is long, each of those features having a # context that overlaps a bit the context of the previous feature. if args.language == 'en': questions = [ examples[i]['question'].encode('ascii', errors='replace').decode('UTF-8') for i in range(len(examples)) ] contexts = [ examples[i]['context'].encode('ascii', errors='replace').decode('UTF-8') for i in range(len(examples)) ] else: questions = [examples[i]['question'] for i in range(len(examples))] contexts = [examples[i]['context'] for i in range(len(examples))] tokenized_examples = tokenizer(questions, contexts, stride=args.doc_stride, max_seq_len=args.max_seq_len) tokenized_examples = convert_tokenizer_res_to_old_version( tokenized_examples) log.debug('\nexample: %d' % len(examples)) log.debug('feature: %d\n' % len(tokenized_examples)) # For validation, there is no need to compute start and end positions for i, tokenized_example in enumerate(tokenized_examples): # Grab the sequence corresponding to that example (to know what is the context and what is the question). sequence_ids = tokenized_example['token_type_ids'] # One example can give several spans, this is the index of the example containing this span of text. sample_index = tokenized_example['overflow_to_sample'] tokenized_examples[i]["example_id"] = examples[sample_index]['id'] tokenized_examples[i]['question'] = examples[sample_index]['question'] tokenized_examples[i]['context'] = examples[sample_index]['context'] tokenized_examples[i]['sent_token'] = examples[sample_index][ 'sent_token'] return tokenized_examples
def map_fn_senti(examples, tokenizer, language): print('load data %d' % len(examples)) if language == 'ch': q_name = "query" t_name = "title" queries = [example[q_name] for example in examples] titles = [example[t_name] for example in examples] else: q_name = "sentence1" t_name = "sentence2" queries = [ example[q_name].encode('ascii', errors='replace').decode('UTF-8') for example in examples ] titles = [ example[t_name].encode('ascii', errors='replace').decode('UTF-8') for example in examples ] tokenized_examples = tokenizer(queries, titles, max_seq_len=args.max_seq_len) tokenized_examples = convert_tokenizer_res_to_old_version( tokenized_examples) for i in range(len(tokenized_examples)): tokenized_examples[i]['query_offset_mapping'] = [ (0, 0) ] + tokenizer.get_offset_mapping( queries[i])[:args.max_seq_len - 2] + [(0, 0)] tokenized_examples[i]['title_offset_mapping'] = [ (0, 0) ] + tokenizer.get_offset_mapping( titles[i])[:args.max_seq_len - 2] + [(0, 0)] return tokenized_examples
def map_fn_DuCheckList_finetune(examples): # Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results # in one example possible giving several features when a context is long, each of those features having a # context that overlaps a bit the context of the previous feature. questions = [examples[i]['question'] for i in range(len(examples))] contexts = [ examples[i]['context'] + examples[i]['title'] for i in range(len(examples)) ] tokenized_examples = tokenizer( questions, contexts, stride=args.doc_stride, max_seq_len=args.max_seq_len) tokenized_examples = convert_tokenizer_res_to_old_version( tokenized_examples) for i, tokenized_example in enumerate(tokenized_examples): # We will label impossible answers with the index of the CLS token. input_ids = tokenized_example["input_ids"] # list(seq) cls_index = input_ids.index(tokenizer.cls_token_id) # Grab the sequence corresponding to that example (to know what is the context and what is the question). sequence_ids = tokenized_example['token_type_ids'] # list(seq) # The offset mappings will give us a map from token to character position in the original context. This will # help us compute the start_positions and end_positions. offsets = tokenized_example['offset_mapping'] # list(seq) # One example can give several spans, this is the index of the example containing this span of text. sample_index = tokenized_example['overflow_to_sample'] # int if args.language == 'ch': answers = examples[sample_index]['answers'] # list answer_starts = examples[sample_index]['answer_starts'] # list else: example = examples[sample_index] example['question_len'] = len(example['question'].split()) example['context_len'] = len(example['context'].split()) answers = example['answers'] # list answer_starts = example['answer_starts'] # list # If no answers are given, set the cls_index as answer. if len(answer_starts) == 0: tokenized_examples[i]["start_positions"] = cls_index tokenized_examples[i]["end_positions"] = cls_index tokenized_examples[i]['answerable_label'] = 0 else: # Start/end character index of the answer in the text. start_char = answer_starts[0] end_char = start_char + len(answers[0]) if args.language == 'ch': # Start token index of the current span in the text. token_start_index = 0 while sequence_ids[token_start_index] != 1: token_start_index += 1 # End token index of the current span in the text. token_end_index = len(input_ids) - 2 while sequence_ids[token_end_index] != 1: token_end_index -= 1 else: token_start_index = tokenized_example['context_start_id'] token_end_index = tokenized_example['context_end_id'] # Detect if the answer is out of the span (in which case this feature is labeled with the CLS index). if not (offsets[token_start_index][0] <= start_char and offsets[token_end_index][1] >= end_char): tokenized_examples[i]["start_positions"] = cls_index tokenized_examples[i]["end_positions"] = cls_index tokenized_examples[i]['answerable_label'] = 0 else: # Otherwise move the token_start_index and token_end_index to the two ends of the answer. # Note: we could go after the last offset if the answer is the last word (edge case). while token_start_index < len(offsets) and offsets[ token_start_index][0] <= start_char: token_start_index += 1 tokenized_examples[i]["start_positions"] = token_start_index - 1 while offsets[token_end_index][1] >= end_char: token_end_index -= 1 tokenized_examples[i]["end_positions"] = token_end_index + 1 tokenized_examples[i]['answerable_label'] = 1 return tokenized_examples