def test_one_line_without_leftover(self):
with open('test_files/one_line') as f:
g = extract_x_without_overlap_y_shifted_by_1(opened_file=f,
seq_len=7)
res = list(g)
self.assertEqual(
res,
[(['WONDER', 'HOW', 'MUCH', 'OF', 'THE', 'MEETINGS', 'IS'
], ['HOW', 'MUCH', 'OF', 'THE', 'MEETINGS', 'IS', 'TALKING']),
(['TALKING', 'ABOUT', 'THE', 'STUFF', 'AT', 'THE', 'MEETINGS'
], ['ABOUT', 'THE', 'STUFF', 'AT', 'THE', 'MEETINGS', '</s>'])])
def test_short_lines_with_leftover(self):
with open('test_files/short_lines') as f:
g = extract_x_without_overlap_y_shifted_by_1(opened_file=f,
seq_len=7)
res = list(g)
self.assertEqual(
res,
[(['WONDER', 'HOW', 'MUCH', 'OF', 'THE', 'MEETINGS', 'IS'
], ['HOW', 'MUCH', 'OF', 'THE', 'MEETINGS', 'IS', 'TALKING']),
(['TALKING', 'ABOUT', 'THE', 'STUFF', 'AT', 'THE', 'MEETINGS'
], ['ABOUT', 'THE', 'STUFF', 'AT', 'THE', 'MEETINGS', '</s>']),
(['</s>', 'YEAH', '</s>', 'NOT', 'A', 'LOT', '</s>'
], ['YEAH', '</s>', 'NOT', 'A', 'LOT', '</s>', 'NO']),
(['NO', '</s>', 'HMM', '</s>', 'OKAY', '</s>', 'SOUNDS'
], ['</s>', 'HMM', '</s>', 'OKAY', '</s>', 'SOUNDS', 'LIKE'])])
def test_one_line_seq_len_bigger_than_words_in_file(self):
with open('test_files/one_line') as f:
g = extract_x_without_overlap_y_shifted_by_1(opened_file=f,
seq_len=16)
res = list(g)
self.assertEqual(res, [])
def test_empty_file_with_seq_len(self):
with open('test_files/empty') as f:
g = extract_x_without_overlap_y_shifted_by_1(opened_file=f,
seq_len=2)
res = list(g)
self.assertEqual(len(res), 0)