def test_readline(self):
blob = mock.Mock()
def read_from_fake_data(start=0, end=None, **_):
return TEST_BINARY_DATA[start:end]
blob.download_as_bytes = mock.Mock(side_effect=read_from_fake_data)
reader = BlobReader(blob, chunk_size=10)
# Read a line. With chunk_size=10, expect three chunks downloaded.
self.assertEqual(reader.readline(), TEST_BINARY_DATA[:27])
blob.download_as_bytes.assert_called_with(start=20, end=30, checksum=None)
self.assertEqual(blob.download_as_bytes.call_count, 3)
# Read another line.
self.assertEqual(reader.readline(), TEST_BINARY_DATA[27:])
blob.download_as_bytes.assert_called_with(start=50, end=60, checksum=None)
self.assertEqual(blob.download_as_bytes.call_count, 6)
blob.size = len(TEST_BINARY_DATA)
reader.seek(0)
# Read all lines. The readlines algorithm will attempt to read past the end of the last line once to verify there is no more to read.
self.assertEqual(b"".join(reader.readlines()), TEST_BINARY_DATA)
blob.download_as_bytes.assert_called_with(
start=len(TEST_BINARY_DATA), end=len(TEST_BINARY_DATA) + 10, checksum=None
)
self.assertEqual(blob.download_as_bytes.call_count, 13)
reader.close()
def test_close(self):
blob = mock.Mock()
reader = BlobReader(blob)
reader.close()
with self.assertRaises(ValueError):
reader.read()
with self.assertRaises(ValueError):
reader.seek(0)
def test_read(self):
blob = mock.Mock()
def read_from_fake_data(start=0, end=None, **_):
return TEST_BINARY_DATA[start:end]
blob.download_as_bytes = mock.Mock(side_effect=read_from_fake_data)
download_kwargs = {"if_metageneration_match": 1}
reader = BlobReader(blob, chunk_size=8, **download_kwargs)
# Read and trigger the first download of chunk_size.
self.assertEqual(reader.read(1), TEST_BINARY_DATA[0:1])
blob.download_as_bytes.assert_called_once_with(start=0,
end=8,
checksum=None,
retry=DEFAULT_RETRY,
**download_kwargs)
# Read from buffered data only.
self.assertEqual(reader.read(3), TEST_BINARY_DATA[1:4])
blob.download_as_bytes.assert_called_once()
# Read remaining buffer plus an additional chunk read.
self.assertEqual(reader.read(8), TEST_BINARY_DATA[4:12])
self.assertEqual(reader._pos, 12)
self.assertEqual(blob.download_as_bytes.call_count, 2)
blob.download_as_bytes.assert_called_with(start=8,
end=16,
checksum=None,
retry=DEFAULT_RETRY,
**download_kwargs)
# Read a larger amount, requiring a download larger than chunk_size.
self.assertEqual(reader.read(16), TEST_BINARY_DATA[12:28])
self.assertEqual(reader._pos, 28)
self.assertEqual(blob.download_as_bytes.call_count, 3)
blob.download_as_bytes.assert_called_with(start=16,
end=28,
checksum=None,
retry=DEFAULT_RETRY,
**download_kwargs)
# Read all remaining data.
self.assertEqual(reader.read(), TEST_BINARY_DATA[28:])
self.assertEqual(blob.download_as_bytes.call_count, 4)
blob.download_as_bytes.assert_called_with(start=28,
end=None,
checksum=None,
retry=DEFAULT_RETRY,
**download_kwargs)
reader.close()
def test_seek(self):
blob = mock.Mock()
def read_from_fake_data(start=0, end=None, **_):
return TEST_BINARY_DATA[start:end]
blob.download_as_bytes = mock.Mock(side_effect=read_from_fake_data)
blob.size = None
download_kwargs = {"if_metageneration_match": 1}
reader = BlobReader(blob, chunk_size=8, **download_kwargs)
# Seek needs the blob size to work and should call reload() if the size
# is not known. Set a mock to initialize the size if reload() is called.
def initialize_size(**_):
blob.size = len(TEST_BINARY_DATA)
blob.reload = mock.Mock(side_effect=initialize_size)
# Seek, forcing a blob reload in order to validate the seek doesn't
# exceed the end of the blob.
self.assertEqual(reader.seek(4), 4)
blob.reload.assert_called_once_with(**download_kwargs)
self.assertEqual(reader.read(4), TEST_BINARY_DATA[4:8])
self.assertEqual(blob.download_as_bytes.call_count, 1)
# Seek forward 2 bytes with whence=1. Position is still in buffer.
self.assertEqual(reader.seek(2, 1), 10)
self.assertEqual(reader.read(2), TEST_BINARY_DATA[10:12])
self.assertEqual(blob.download_as_bytes.call_count, 1)
# Attempt seek past end of file. Position should be at end of file.
self.assertEqual(
reader.seek(len(TEST_BINARY_DATA) + 100), len(TEST_BINARY_DATA)
)
# Seek to beginning. The next read will need to download data again.
self.assertEqual(reader.seek(0), 0)
self.assertEqual(reader.read(4), TEST_BINARY_DATA[0:4])
self.assertEqual(blob.download_as_bytes.call_count, 2)
# Seek relative to end with whence=2.
self.assertEqual(reader.seek(-1, 2), len(TEST_BINARY_DATA) - 1)
self.assertEqual(reader.read(), TEST_BINARY_DATA[-1:])
self.assertEqual(blob.download_as_bytes.call_count, 3)
with self.assertRaises(ValueError):
reader.seek(1, 4)
# tell() is an inherited method that uses seek().
self.assertEqual(reader.tell(), reader._pos)
reader.close()
def test_retry_passed_through(self):
blob = mock.Mock()
def read_from_fake_data(start=0, end=None, **_):
return TEST_BINARY_DATA[start:end]
blob.download_as_bytes = mock.Mock(side_effect=read_from_fake_data)
download_kwargs = {"if_metageneration_match": 1}
reader = BlobReader(blob, chunk_size=8, retry=None, **download_kwargs)
# Read and trigger the first download of chunk_size.
self.assertEqual(reader.read(1), TEST_BINARY_DATA[0:1])
blob.download_as_bytes.assert_called_once_with(start=0,
end=8,
checksum=None,
retry=None,
**download_kwargs)
reader.close()