def test_read_delimited_chunks_empty_input(self):
"""Test empty input for read_delimited_chunks."""
# splittign an empty file yields no chunks at all, similar to
# how a string's readline method yields an empty list for an
# empty string.
self.assertEqual(list(read_delimited_chunks(io.BytesIO(b""), 1000)),
[])
def test_read_delimited_chunks_long_lines(self):
"""Test lines longer than chunksize, data longer than chunk_size."""
chunk_size = 100
# Make sure the test data fits the testcase:
#
# 1. At least one lines must be longer than the chunk_size, so
# that at least one chunk has to be longer as well
self.assertTrue(max(map(len, csv_test_data.splitlines())) > chunk_size)
# The actual test
chunked = list(
read_delimited_chunks(io.BytesIO(csv_test_data), chunk_size))
# Appending all chunks yields the original unchunked data
self.assertEqual(csv_test_data, b"".join(chunked))
# Chunks longer than chunk_size have newlines only at the end.
long_chunks = list(
filter(lambda chunk: len(chunk) > chunk_size, chunked))
self.assertTrue(len(long_chunks) > 0)
for chunk in long_chunks:
if chunk.endswith(b"\n"):
chunk = chunk[:-1]
self.assertFalse(b"\n" in chunk)
def test_read_delimited_chunks_short_lines(self):
"""Test lines shorter than chunksize, data longer than chunk_size."""
chunk_size = 500
# Make sure the test data fits the testcase:
#
# 1. The lines must be shorter than the chunk_size, so that all
# chunks can be shorter than chunk_size
self.assertTrue(max(map(len, csv_test_data.splitlines())) < chunk_size)
# 2. The data must be longer than chunk_size so that chunking is
# useful
self.assertTrue(len(csv_test_data) > chunk_size)
# The actual test
chunked = list(
read_delimited_chunks(io.BytesIO(csv_test_data), chunk_size))
# Appending all chunks yields the original unchunked data
self.assertEqual(csv_test_data, b"".join(chunked))
# All chunks are shorter than chunk_size
self.assertTrue(max(map(len, chunked)) <= chunk_size)
# All chunks were split at line separators
self.assertEqual(
csv_test_data.splitlines(),
list(
itertools.chain.from_iterable(chunk.splitlines()
for chunk in chunked)))
def test_read_delimited_chunks_long_lines(self):
"""Test lines longer than chunksize, data longer than chunk_size."""
chunk_size = 100
# Make sure the test data fits the testcase:
#
# 1. At least one lines must be longer than the chunk_size, so
# that at least one chunk has to be longer as well
self.assertTrue(max(map(len, csv_test_data.splitlines())) > chunk_size)
# The actual test
chunked = list(read_delimited_chunks(io.BytesIO(csv_test_data),
chunk_size))
# Appending all chunks yields the original unchunked data
self.assertEqual(csv_test_data, b"".join(chunked))
# Chunks longer than chunk_size have newlines only at the end.
long_chunks = list(filter(lambda chunk: len(chunk) > chunk_size,
chunked))
self.assertTrue(len(long_chunks) > 0)
for chunk in long_chunks:
if chunk.endswith(b"\n"):
chunk = chunk[:-1]
self.assertFalse(b"\n" in chunk)
def test_read_delimited_chunks_short_lines(self):
"""Test lines shorter than chunksize, data longer than chunk_size."""
chunk_size = 500
# Make sure the test data fits the testcase:
#
# 1. The lines must be shorter than the chunk_size, so that all
# chunks can be shorter than chunk_size
self.assertTrue(max(map(len, csv_test_data.splitlines())) < chunk_size)
# 2. The data must be longer than chunk_size so that chunking is
# useful
self.assertTrue(len(csv_test_data) > chunk_size)
# The actual test
chunked = list(read_delimited_chunks(io.BytesIO(csv_test_data),
chunk_size))
# Appending all chunks yields the original unchunked data
self.assertEqual(csv_test_data, b"".join(chunked))
# All chunks are shorter than chunk_size
self.assertTrue(max(map(len, chunked)) <= chunk_size)
# All chunks were split at line separators
self.assertEqual(csv_test_data.splitlines(),
list(itertools.chain.from_iterable(
chunk.splitlines() for chunk in chunked)))
def test_read_delimited_chunks_empty_input(self):
"""Test empty input for read_delimited_chunks."""
# splittign an empty file yields no chunks at all, similar to
# how a string's readline method yields an empty list for an
# empty string.
self.assertEqual(list(read_delimited_chunks(io.BytesIO(b""),
1000)),
[])