def exponentialBackoff(request: HttpRequest,
time: int = 0,
tries: int = 0,
max_tries: int = 8):
"""Periodically retry a failed request over an increasing amount of time to
handle errors related to rate limits, network volume, or response time.
https://developers.google.com/drive/api/v3/handle-errors
"""
try:
response = None
while response is None:
status, response = request.next_chunk()
if status:
print(
f'Uploading: {round(status.resumable_progress / status.total_size * 100, 2)}%'
)
file = request.execute(num_retries=2)
return file
except HttpError as e:
print(e)
if tries < max_tries:
print(f'Trying again in {time}')
t = Timer(time,
exponentialBackoff,
args=(request, 2**(tries + 1) + random.random(),
tries + 1, max_tries))
t.start()
else:
raise Exception('Max tries reached.')
def googe_doc_upload(filename, filetype, url):
media = MediaFileUpload('files/' + filename, mimetype=filetype)
file_stats = os.stat(filename)
if file_stats.size <= 5e6:
HttpRequest(url, method='POST', body=media).execute()
else:
# Resumable upload
request = HttpRequest(url, method='POST', body=media, resumable=True)
response = None
while response is None:
status, response = request.next_chunk()
def _step_upload(request: HttpRequest):
'''Print the percentage complete for a given upload while it is executing.
@params:
request: HttpRequest, supporting next_chunk() (i.e., is resumable).
@return: tuple(bool, whether or not the upload succeeded
response, the result of the executed request (or None)
@raises: HttpError 417.
This error indicates if the FusionTable's self size limit will be exceeded.
'''
if not request or not isinstance(request, HttpRequest):
return (False, None)
done = None
fails = 0
while done is None:
try:
status, done = request.next_chunk()
except HttpLib2Error as err:
print('Transport error: ', err)
except HttpError as err:
print()
if err.resp.status in [404]:
return (False, None)
if err.resp.status in [500, 502, 503, 504] and fails < 5:
time.sleep(2 ^ fails)
fails += 1
elif (err.resp.status in [417] and fails < 5 and
'Table will exceed allowed maximum size' in err.__str__()):
raise err
else:
print('Upload failed:', err)
return (False, None)
else:
print_progress_bar(status.progress() if status else 1.,
1.,
'Uploading...',
length=50)
print()
return (True, done)
