class TestGcsFileSystem(TestCase):
"""Tests for the GCSFileSystem."""
def setUp(self) -> None:
self.mock_storage_client = create_autospec(storage.Client)
self.fs = GCSFileSystemImpl(self.mock_storage_client)
def test_retry(self) -> None:
mock_bucket = create_autospec(Bucket)
mock_bucket.exists.return_value = True
# Client first raises a Gateway timeout, then returns a normal bucket.
self.mock_storage_client.get_bucket.side_effect = [
exceptions.GatewayTimeout("Exception"),
mock_bucket,
]
# Should not crash!
self.assertTrue(
self.fs.exists(
GcsfsBucketPath.from_absolute_path("gs://my-bucket")))
self.mock_storage_client.bucket.assert_called()
def test_retry_with_fatal_error(self) -> None:
mock_bucket = create_autospec(Bucket)
mock_bucket.exists.return_value = True
# Client first raises a Gateway timeout, then on retry will raise a ValueError
self.mock_storage_client.bucket.side_effect = [
exceptions.GatewayTimeout("Exception"),
ValueError("This will crash"),
]
with self.assertRaises(ValueError):
self.fs.exists(
GcsfsBucketPath.from_absolute_path("gs://my-bucket"))
self.mock_storage_client.bucket.assert_called()
def build(cls) -> GCSFileSystem:
storage_client = storage.Client()
adapter = requests.adapters.HTTPAdapter(
pool_connections=POOL_CONNECTIONS, pool_maxsize=POOL_MAXSIZE)
# pylint: disable=protected-access
storage_client._http.mount("https://", adapter)
storage_client._http._auth_request.session.mount("https://", adapter)
return GCSFileSystemImpl(storage_client)
def setUp(self) -> None:
self.mock_storage_client = create_autospec(storage.Client)
self.fs = GCSFileSystemImpl(self.mock_storage_client)
def build(cls) -> GCSFileSystem:
return GCSFileSystemImpl(storage.Client())