def collate(data, tokenizer, input_block_size,output_block_size):
""" List of tuple as an input. """
inputs=[]
outputs=[]
for i,example in enumerate(data):
input=tokenizer.encode(example.input_text)
input=fit_to_block_size(input, input_block_size, tokenizer.pad_token_id)
inputs.append(input)
if example.output_text is not None:
output=tokenizer.encode(example.output_text)
else:
output=tokenizer.build_inputs_with_special_tokens([])
output=fit_to_block_size(output, output_block_size, tokenizer.pad_token_id)
outputs.append(output)
inputs = torch.tensor(inputs)
outputs = torch.tensor(outputs)
encoder_mask = build_mask(inputs, tokenizer.pad_token_id)
decoder_mask = build_mask(outputs, tokenizer.pad_token_id)
lm_labels = build_lm_labels(outputs, tokenizer.pad_token_id)
return (
inputs,
outputs,
encoder_mask,
decoder_mask,
lm_labels,
)
def collate(data, tokenizer, block_size):
""" List of tuple as an input. """
# remove the files with empty an story/summary, encode and fit to block
data = filter(lambda x: not (len(x[0]) == 0 or len(x[1]) == 0), data)
data = [
encode_for_summarization(story, summary, tokenizer)
for story, summary in data
]
data = [(
fit_to_block_size(story, block_size, tokenizer.pad_token_id),
fit_to_block_size(summary, block_size, tokenizer.pad_token_id),
) for story, summary in data]
stories = torch.tensor([story for story, summary in data])
summaries = torch.tensor([summary for story, summary in data])
encoder_token_type_ids = compute_token_type_ids(stories,
tokenizer.cls_token_id)
encoder_mask = build_mask(stories, tokenizer.pad_token_id)
decoder_mask = build_mask(summaries, tokenizer.pad_token_id)
lm_labels = build_lm_labels(summaries, tokenizer.pad_token_id)
return (
stories,
summaries,
encoder_token_type_ids,
encoder_mask,
decoder_mask,
lm_labels,
)
def collate(data, tokenizer, input_block_size, output_block_size):
""" List of tuple as an input. """
question_inputs = []
question_varible_outputs = []
condition_outputs = []
for i, example in enumerate(data):
question_input = tokenizer.encode(example.question_input)
question_input = fit_to_block_size(question_input, input_block_size,
tokenizer.pad_token_id)
question_inputs.append(question_input)
if example.question_varible_output is not None:
question_varible_output = tokenizer.encode(
example.question_varible_output)
else:
question_varible_output = tokenizer.build_inputs_with_special_tokens(
[])
question_varible_output = fit_to_block_size(question_varible_output,
output_block_size,
tokenizer.pad_token_id)
question_varible_outputs.append(question_varible_output)
if example.condition_output is not None:
condition_output = tokenizer.encode(example.condition_output)
else:
condition_output = tokenizer.build_inputs_with_special_tokens([])
condition_output = fit_to_block_size(condition_output,
output_block_size,
tokenizer.pad_token_id)
condition_outputs.append(condition_output)
question_inputs = torch.tensor(question_inputs)
question_varible_outputs = torch.tensor(question_varible_outputs)
condition_outputs = torch.tensor(condition_outputs)
question_inputs_mask = build_mask(question_inputs, tokenizer.pad_token_id)
question_varible_outputs_mask = build_mask(question_varible_outputs,
tokenizer.pad_token_id)
condition_outputs_mask = build_mask(condition_outputs,
tokenizer.pad_token_id)
question_varible_outputs_mask_lm_labels = build_lm_labels(
question_varible_outputs, tokenizer.pad_token_id)
condition_outputs_mask_lm_labels = build_lm_labels(condition_outputs,
tokenizer.pad_token_id)
return (
question_inputs,
[question_varible_outputs, condition_outputs],
question_inputs_mask,
[question_varible_outputs_mask, condition_outputs_mask],
[
question_varible_outputs_mask_lm_labels,
condition_outputs_mask_lm_labels
],
)
def test_fit_to_block_sequence_too_big(self):
""" Truncate the sequence if it is too long. """
sequence = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
expected_output = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
self.assertEqual(
fit_to_block_size(sequence, self.block_size, 0), expected_output
)
def test_fit_to_block_sequence_fit_exactly(self):
""" Do nothing if the sequence is the right size. """
sequence = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
expected_output = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
self.assertEqual(
fit_to_block_size(sequence, self.block_size, 0), expected_output
)
def test_fit_to_block_sequence_too_small(self):
""" Pad the sequence with 0 if the sequence is smaller than the block size."""
sequence = [1, 2, 3, 4]
expected_output = [1, 2, 3, 4, 0, 0, 0, 0, 0, 0]
self.assertEqual(
fit_to_block_size(sequence, self.block_size, 0), expected_output
)
def collate(data, tokenizer, block_size, device):
""" Collate formats the data passed to the data loader.
In particular we tokenize the data batch after batch to avoid keeping them
all in memory. We output the data as a namedtuple to fit the original BertAbs's
API.
"""
data = [x for x in data if not len(x[1]) == 0] # remove empty_files
names = [name for name, _, _ in data]
summaries = [" ".join(summary_list) for _, _, summary_list in data]
encoded_text = [
encode_for_summarization(story, summary, tokenizer)
for _, story, summary in data
]
encoded_stories = torch.tensor([
fit_to_block_size(story, block_size, tokenizer.pad_token_id)
for story, _ in encoded_text
])
encoder_token_type_ids = compute_token_type_ids(encoded_stories,
tokenizer.cls_token_id)
encoder_mask = build_mask(encoded_stories, tokenizer.pad_token_id)
batch = Batch(
document_names=names,
batch_size=len(encoded_stories),
src=encoded_stories.to(device),
segs=encoder_token_type_ids.to(device),
mask_src=encoder_mask.to(device),
tgt_str=summaries,
)
return batch
def collate(data, encoder_tokenizer, decoder_tokenizer, input_block_size,
output_block_size):
""" List of tuple as an input. """
inputs = []
outputs = []
vocabs = []
example_buffer = []
for i, example in enumerate(data):
#input=encoder_tokenizer.encode(example.input)
example_buffer.append(example)
tok_to_orig_index = []
orig_to_tok_index = []
all_doc_tokens = []
input_tokens = ['[CLS]'] + example.input.split() + ['SEP']
for (i, token) in enumerate(input_tokens):
orig_to_tok_index.append(len(all_doc_tokens))
sub_tokens = encoder_tokenizer.tokenize(token)
for sub_token in sub_tokens:
tok_to_orig_index.append(i)
all_doc_tokens.append(sub_token)
input = encoder_tokenizer.convert_tokens_to_ids(all_doc_tokens)
example.tok_to_orig_index = tok_to_orig_index
example.orig_to_tok_index = orig_to_tok_index
input = fit_to_block_size(input, input_block_size,
encoder_tokenizer.pad_token_id)
inputs.append(input)
if example.output is not None:
#output=tokenizer.encode(example.output)
output_tokens = example.output.split()
#print('Before Whole Index: {}'.format(output_tokens))
#print('encoder input: {}'.format(all_doc_tokens))
output_tokens = translate_tokenindex_to_subtokenindex(
example, output_tokens, example.vocab_indexes,
example.fsa_states)
#print('After Sub Index: {}'.format(output_tokens))
output = decoder_tokenizer.convert_tokens_to_ids(output_tokens)
output_states = example.fsa_states
else:
#output=decoder_tokenizer.build_inputs_with_special_tokens(['start'])
output = decoder_tokenizer.convert_tokens_to_ids(['start'])
output_vocab_indexes = example.vocab_indexes
output = fit_to_block_size(output, output_block_size,
decoder_tokenizer.pad_token_id)
output_vocab_indexes = fit_to_block_size(
output_vocab_indexes, output_block_size,
decoder_tokenizer.pad_token_id)
outputs.append(output)
vocabs.append(output_vocab_indexes)
#print('debug output={}'.format(example.output.split()))
#print('debug output_states={}'.format(output_states))
#print('debug output_vocab_indexes={}'.format(output_vocab_indexes))
#print('debug outputid={}'.format(output))
#if example.vocab_indexes is not None:
# vocab=example.vocab_indexes
#else:
# vocab=[1]
#vocabs.append(vocab)
#print(tokenizer.vocab)
#exit(-1)
inputs = torch.tensor(inputs)
outputs = torch.tensor(outputs)
vocabs = torch.tensor(vocabs)
inputs_mask = build_mask(inputs, encoder_tokenizer.pad_token_id)
outputs_mask = build_mask(outputs, decoder_tokenizer.pad_token_id)
vocabs_mask = build_mask(vocabs, decoder_tokenizer.pad_token_id)
outputs_mask_lm_labels = build_lm_labels(outputs,
decoder_tokenizer.pad_token_id)
vocabs_mask_lm_labels = build_lm_labels(vocabs,
decoder_tokenizer.pad_token_id)
return (
inputs,
outputs,
vocabs,
inputs_mask,
outputs_mask,
vocabs_mask,
outputs_mask_lm_labels,
vocabs_mask_lm_labels,
example_buffer,
)
