def test_416_error_handled(self):
blob = mock.Mock()
blob.download_as_bytes = mock.Mock(
side_effect=RequestRangeNotSatisfiable("message"))
reader = BlobReader(blob)
self.assertEqual(reader.read(), b"")
def test_attributes(self):
blob = mock.Mock()
blob.chunk_size = 256
reader = BlobReader(blob)
self.assertTrue(reader.seekable())
self.assertTrue(reader.readable())
self.assertFalse(reader.writable())
self.assertEqual(256, reader._chunk_size)
def test_attributes(self):
blob = mock.Mock()
blob.chunk_size = 256
reader = BlobReader(blob)
self.assertTrue(reader.seekable())
self.assertTrue(reader.readable())
self.assertFalse(reader.writable())
self.assertEqual(reader._chunk_size, 256)
self.assertEqual(reader._retry, DEFAULT_RETRY)
def test_readline(self):
blob = mock.Mock()
def read_from_fake_data(start=0, end=None, **_):
return TEST_BINARY_DATA[start:end]
blob.download_as_bytes = mock.Mock(side_effect=read_from_fake_data)
reader = BlobReader(blob, chunk_size=10)
# Read a line. With chunk_size=10, expect three chunks downloaded.
self.assertEqual(reader.readline(), TEST_BINARY_DATA[:27])
blob.download_as_bytes.assert_called_with(start=20, end=30, checksum=None)
self.assertEqual(blob.download_as_bytes.call_count, 3)
# Read another line.
self.assertEqual(reader.readline(), TEST_BINARY_DATA[27:])
blob.download_as_bytes.assert_called_with(start=50, end=60, checksum=None)
self.assertEqual(blob.download_as_bytes.call_count, 6)
blob.size = len(TEST_BINARY_DATA)
reader.seek(0)
# Read all lines. The readlines algorithm will attempt to read past the end of the last line once to verify there is no more to read.
self.assertEqual(b"".join(reader.readlines()), TEST_BINARY_DATA)
blob.download_as_bytes.assert_called_with(
start=len(TEST_BINARY_DATA), end=len(TEST_BINARY_DATA) + 10, checksum=None
)
self.assertEqual(blob.download_as_bytes.call_count, 13)
reader.close()
def test_multibyte_seek(self):
blob = mock.Mock()
def read_from_fake_data(start=0, end=None, **_):
return TEST_MULTIBYTE_TEXT_DATA.encode("utf-8")[start:end]
blob.download_as_bytes = mock.Mock(side_effect=read_from_fake_data)
blob.size = None
blob.chunk_size = None
download_kwargs = {"if_metageneration_match": 1}
reader = io.TextIOWrapper(BlobReader(blob, **download_kwargs))
# Seek needs the blob size to work and should call reload() if the size
# is not known. Set a mock to initialize the size if reload() is called.
def initialize_size(**_):
blob.size = len(TEST_MULTIBYTE_TEXT_DATA.encode("utf-8"))
blob.reload = mock.Mock(side_effect=initialize_size)
# Seek, forcing a blob reload in order to validate the seek doesn't
# exceed the end of the blob.
self.assertEqual(reader.seek(4), 4)
blob.reload.assert_called_once_with(**download_kwargs)
# Seek to beginning.
self.assertEqual(reader.seek(0), 0)
self.assertEqual(reader.read(), TEST_MULTIBYTE_TEXT_DATA)
self.assertEqual(blob.download_as_bytes.call_count, 1)
# tell() is an inherited method that uses seek().
self.assertEqual(reader.tell(), len(TEST_MULTIBYTE_TEXT_DATA.encode("utf-8")))
reader.close()
def test_multibyte_read(self):
blob = mock.Mock()
def read_from_fake_data(start=0, end=None, **_):
return TEST_MULTIBYTE_TEXT_DATA.encode("utf-8")[start:end]
blob.download_as_bytes = mock.Mock(side_effect=read_from_fake_data)
blob.chunk_size = None
blob.size = len(TEST_MULTIBYTE_TEXT_DATA.encode("utf-8"))
download_kwargs = {"if_metageneration_match": 1}
reader = io.TextIOWrapper(BlobReader(blob, **download_kwargs))
# The TextIOWrapper class has an internally defined chunk size which
# will override ours. The wrapper class is not under test.
# Read and trigger the first download of chunk_size.
self.assertEqual(reader.read(1), TEST_MULTIBYTE_TEXT_DATA[0:1])
blob.download_as_bytes.assert_called_once()
# Read from buffered data only.
self.assertEqual(reader.read(3), TEST_MULTIBYTE_TEXT_DATA[1:4])
blob.download_as_bytes.assert_called_once()
# Read all remaining data.
self.assertEqual(reader.read(), TEST_MULTIBYTE_TEXT_DATA[4:])
# Seek to 0 and read all remaining data again.
reader.seek(0)
self.assertEqual(reader.read(), TEST_MULTIBYTE_TEXT_DATA)
reader.close()
def test_read(self):
blob = mock.Mock()
def read_from_fake_data(start=0, end=None, **_):
return TEST_BINARY_DATA[start:end]
blob.download_as_bytes = mock.Mock(side_effect=read_from_fake_data)
download_kwargs = {"if_metageneration_match": 1}
reader = BlobReader(blob, chunk_size=8, **download_kwargs)
# Read and trigger the first download of chunk_size.
self.assertEqual(reader.read(1), TEST_BINARY_DATA[0:1])
blob.download_as_bytes.assert_called_once_with(start=0,
end=8,
checksum=None,
retry=DEFAULT_RETRY,
**download_kwargs)
# Read from buffered data only.
self.assertEqual(reader.read(3), TEST_BINARY_DATA[1:4])
blob.download_as_bytes.assert_called_once()
# Read remaining buffer plus an additional chunk read.
self.assertEqual(reader.read(8), TEST_BINARY_DATA[4:12])
self.assertEqual(reader._pos, 12)
self.assertEqual(blob.download_as_bytes.call_count, 2)
blob.download_as_bytes.assert_called_with(start=8,
end=16,
checksum=None,
retry=DEFAULT_RETRY,
**download_kwargs)
# Read a larger amount, requiring a download larger than chunk_size.
self.assertEqual(reader.read(16), TEST_BINARY_DATA[12:28])
self.assertEqual(reader._pos, 28)
self.assertEqual(blob.download_as_bytes.call_count, 3)
blob.download_as_bytes.assert_called_with(start=16,
end=28,
checksum=None,
retry=DEFAULT_RETRY,
**download_kwargs)
# Read all remaining data.
self.assertEqual(reader.read(), TEST_BINARY_DATA[28:])
self.assertEqual(blob.download_as_bytes.call_count, 4)
blob.download_as_bytes.assert_called_with(start=28,
end=None,
checksum=None,
retry=DEFAULT_RETRY,
**download_kwargs)
reader.close()
def test_close(self):
blob = mock.Mock()
reader = BlobReader(blob)
reader.close()
with self.assertRaises(ValueError):
reader.read()
with self.assertRaises(ValueError):
reader.seek(0)
def test_retry_passed_through(self):
blob = mock.Mock()
def read_from_fake_data(start=0, end=None, **_):
return TEST_BINARY_DATA[start:end]
blob.download_as_bytes = mock.Mock(side_effect=read_from_fake_data)
download_kwargs = {"if_metageneration_match": 1}
reader = BlobReader(blob, chunk_size=8, retry=None, **download_kwargs)
# Read and trigger the first download of chunk_size.
self.assertEqual(reader.read(1), TEST_BINARY_DATA[0:1])
blob.download_as_bytes.assert_called_once_with(start=0,
end=8,
checksum=None,
retry=None,
**download_kwargs)
reader.close()
def test_attributes(self):
blob = mock.Mock()
reader = io.TextIOWrapper(BlobReader(blob))
self.assertTrue(reader.seekable())
self.assertTrue(reader.readable())
self.assertFalse(reader.writable())
def test_rejects_invalid_kwargs(self):
blob = mock.Mock()
with self.assertRaises(ValueError):
BlobReader(blob, invalid_kwarg=1)
def test_context_mgr(self):
# Just very that the context manager form doesn't crash.
blob = mock.Mock()
with BlobReader(blob) as reader:
reader.close()
def test_seek(self):
blob = mock.Mock()
def read_from_fake_data(start=0, end=None, **_):
return TEST_BINARY_DATA[start:end]
blob.download_as_bytes = mock.Mock(side_effect=read_from_fake_data)
blob.size = None
download_kwargs = {"if_metageneration_match": 1}
reader = BlobReader(blob, chunk_size=8, **download_kwargs)
# Seek needs the blob size to work and should call reload() if the size
# is not known. Set a mock to initialize the size if reload() is called.
def initialize_size(**_):
blob.size = len(TEST_BINARY_DATA)
blob.reload = mock.Mock(side_effect=initialize_size)
# Seek, forcing a blob reload in order to validate the seek doesn't
# exceed the end of the blob.
self.assertEqual(reader.seek(4), 4)
blob.reload.assert_called_once_with(**download_kwargs)
self.assertEqual(reader.read(4), TEST_BINARY_DATA[4:8])
self.assertEqual(blob.download_as_bytes.call_count, 1)
# Seek forward 2 bytes with whence=1. Position is still in buffer.
self.assertEqual(reader.seek(2, 1), 10)
self.assertEqual(reader.read(2), TEST_BINARY_DATA[10:12])
self.assertEqual(blob.download_as_bytes.call_count, 1)
# Attempt seek past end of file. Position should be at end of file.
self.assertEqual(
reader.seek(len(TEST_BINARY_DATA) + 100), len(TEST_BINARY_DATA)
)
# Seek to beginning. The next read will need to download data again.
self.assertEqual(reader.seek(0), 0)
self.assertEqual(reader.read(4), TEST_BINARY_DATA[0:4])
self.assertEqual(blob.download_as_bytes.call_count, 2)
# Seek relative to end with whence=2.
self.assertEqual(reader.seek(-1, 2), len(TEST_BINARY_DATA) - 1)
self.assertEqual(reader.read(), TEST_BINARY_DATA[-1:])
self.assertEqual(blob.download_as_bytes.call_count, 3)
with self.assertRaises(ValueError):
reader.seek(1, 4)
# tell() is an inherited method that uses seek().
self.assertEqual(reader.tell(), reader._pos)
reader.close()
def _make_blob_reader(*args, **kwargs):
from google.cloud.storage.fileio import BlobReader
return BlobReader(*args, **kwargs)
def test_attributes_explict(self):
blob = mock.Mock()
blob.chunk_size = 256
reader = BlobReader(blob, chunk_size=1024, retry=None)
self.assertEqual(reader._chunk_size, 1024)
self.assertIsNone(reader._retry)
