def testMultipleExceptions(self):
class FooError(Exception):
pass
class BarError(Exception):
pass
class BazError(Exception):
pass
opts = retry.Opts()
opts.attempts = 4
opts.sleep = lambda _: None
counter = []
@retry.On((FooError, BarError, BazError), opts=opts)
def Func() -> None:
counter.append(())
if counter == 1:
raise FooError()
if counter == 2:
raise BarError()
if counter == 3:
raise BazError()
Func() # Should not raise.
def testImmediateSuccess(self):
opts = retry.Opts()
opts.attempts = 1
opts.sleep = lambda _: None
@retry.On((), opts=opts)
def Func() -> None:
pass
Func() # Should not raise.
def testRetriedSuccess(self):
opts = retry.Opts()
opts.attempts = 3
opts.sleep = lambda _: None
counter = []
@retry.On((RuntimeError, ), opts=opts)
def Func() -> None:
counter.append(())
if len(counter) < 3:
raise RuntimeError()
Func() # Should not raise.
def testRetriedFailure(self):
opts = retry.Opts()
opts.attempts = 3
opts.sleep = lambda _: None
counter = []
@retry.On((RuntimeError, ), opts=opts)
def Func() -> None:
counter.append(())
if len(counter) < 4:
raise RuntimeError()
with self.assertRaises(RuntimeError):
Func()
def testRetriedArgumentAndResult(self):
opts = retry.Opts()
opts.attempts = 3
opts.sleep = lambda _: None
counter = []
@retry.On((RuntimeError, ), opts=opts)
def Func(left: str, right: str) -> int:
counter.append(())
if len(counter) < 3:
raise RuntimeError()
return int(f"{left}{right}")
self.assertEqual(Func("13", "37"), 1337)
def testInitDelay(self):
delays = []
opts = retry.Opts()
opts.attempts = 4
opts.init_delay_secs = 42.0
opts.backoff = 1.0
opts.sleep = delays.append
@retry.On((RuntimeError, ), opts=opts)
def Func() -> None:
raise RuntimeError()
with self.assertRaises(RuntimeError):
Func()
self.assertEqual(delays, [42.0, 42.0, 42.0])
def testBackoff(self):
delays = []
opts = retry.Opts()
opts.attempts = 7
opts.init_delay_secs = 1.0
opts.backoff = 2.0
opts.sleep = delays.append
@retry.On((RuntimeError, ), opts=opts)
def Func() -> None:
raise RuntimeError()
with self.assertRaises(RuntimeError):
Func()
self.assertEqual(delays, [1.0, 2.0, 4.0, 8.0, 16.0, 32.0])
def testMaxDelay(self):
delays = []
opts = retry.Opts()
opts.attempts = 6
opts.init_delay_secs = 1.0
opts.max_delay_secs = 3.0
opts.backoff = 1.5
opts.sleep = delays.append
@retry.On((RuntimeError, ), opts=opts)
def Func() -> None:
raise RuntimeError()
with self.assertRaises(RuntimeError):
Func()
self.assertEqual(delays, [1.0, 1.5, 2.25, 3.0, 3.0])
def testNegativeTries(self):
opts = retry.Opts()
opts.attempts = -1
with self.assertRaisesRegex(ValueError, "number of retries"):
retry.On((), opts=opts)
def SendFile(self, file: IO[bytes], opts: Optional[Opts] = None) -> None:
"""Streams the given file to Google Cloud Storage.
Args:
file: A file-like object to send.
opts: Options used for the transfer procedure.
Returns:
Nothing.
Raises:
RequestError: If it is not possible to deliver one of the chunks.
ResponseError: If the server responded with unexpected status.
"""
if opts is None:
opts = self.Opts()
def Sleep(secs: float) -> None:
time.Sleep(
secs,
progress_secs=opts.progress_interval,
progress_callback=opts.progress_callback)
retry_opts = retry.Opts()
retry_opts.attempts = opts.retry_chunk_attempts
retry_opts.backoff = opts.retry_chunk_backoff
retry_opts.init_delay_secs = opts.retry_chunk_init_delay
retry_opts.max_delay_secs = opts.retry_chunk_max_delay
retry_opts.sleep = Sleep
offset = 0
while True:
chunk = file.read(opts.chunk_size)
# To determine whether we are at the last chunk we simply check whether
# the amount of bytes actually read is less than the requested amount.
# Note that this still works if the last chunk is exactly the requested
# size in which case the next tick of the loop will send an empty packet
# finishing the procedure.
is_last_chunk = len(chunk) < opts.chunk_size
# The chunk content range according in the HTTP 1.1 syntax [1].
#
# During the resumable upload procedure the total size of the file might
# not be known upfront. Fortunately, it is only required to send the total
# size only with the last chunk.
#
# Because the range is inclusive in general there is no proper way of
# dealing with empty ranges. This can happen if the file we attempt to
# send is empty or the total number of bytes of the file is divisible by
# the chunk size. In such a case we simply make first and last byte equal
# and hope that the content length header set to 0 is enough to let the
# server figure out that this range is in face empty (which seems to be
# the case).
#
# [1]: https://tools.ietf.org/html/rfc7233#section-4.2
chunk_first_byte = offset
chunk_last_byte = max(offset + len(chunk) - 1, offset)
if is_last_chunk:
total_size = offset + len(chunk)
chunk_range = f"bytes {chunk_first_byte}-{chunk_last_byte}/{total_size}"
else:
chunk_range = f"bytes {chunk_first_byte}-{chunk_last_byte}/*"
headers = {
"Content-Length": str(len(chunk)),
"Content-Range": chunk_range,
}
# We attempt to retry sending chunks in two scenarios: either we failed to
# send the request at all (e.g. due to the internet being down) or because
# there was an interruption error (it is not completely clear when exactly
# this can occur, but the documentation clearly states that the upload can
# be resumed in such a case).
#
# We should not attempt to retry any other errors. If a response is not
# correct or expected it likely means that something is seriously wrong
# and it is better to fail and notify the flow about the problem. It is
# also possible that the upload process has been cancelled (i.e. by the
# analyst), in which case further attempts to send the file are futile.
@retry.On((RequestError, InterruptedResponseError), opts=retry_opts)
def PutChunk():
try:
# We need to set a timeout equal to the requested progress function
# call interval so that we can call it often enough. In general, if an
# internet connection is so bad that these requests take more than the
# frequency of the progress calls, we likely won't be able to send big
# files in reasonable amount of time anyway.
response = requests.put(
self.uri,
data=chunk,
headers=headers,
timeout=opts.progress_interval)
except exceptions.RequestException as error:
raise RequestError("Chunk transmission failure") from error
if 500 <= response.status_code <= 599:
raise InterruptedResponseError(response)
if 400 <= response.status_code <= 499:
raise ResponseError("Cancelled upload session", response)
if is_last_chunk and response.status_code not in [200, 201]:
raise ResponseError("Unexpected final chunk response", response)
if not is_last_chunk and response.status_code != 308:
raise ResponseError("Unexpected mid chunk response", response)
PutChunk()
# TODO: Add support for more detailed progress updates that
# would include state of the upload.
opts.progress_callback()
if is_last_chunk:
break
else:
offset += len(chunk)