def upload_file(self, url, file, callback=None, extra_headers={}):
"""Uploads a file to W&B with failure resumption
Args:
url (str): The url to download
file (str): The path to the file you want to upload
callback (:obj:`func`, optional): A callback which is passed the number of
bytes uploaded since the last time it was called, used to report progress
Returns:
The requests library response object
"""
extra_headers = extra_headers.copy()
response = None
progress = Progress(file, callback=callback)
if progress.len == 0:
raise CommError("%s is an empty file" % file.name)
try:
response = requests.put(
url, data=progress, headers=extra_headers)
response.raise_for_status()
except requests.exceptions.RequestException as e:
status_code = e.response.status_code if e.response != None else 0
# Retry errors from cloud storage or local network issues
if status_code in (308, 409, 429, 500, 502, 503, 504) or isinstance(e, (requests.exceptions.Timeout, requests.exceptions.ConnectionError)):
util.sentry_reraise(retry.TransientException(exc=e))
else:
util.sentry_reraise(e)
return response
def upload_file(self, url, file, callback=None, extra_headers={}):
"""Uploads a file to W&B with failure resumption
Args:
url (str): The url to download
file (str): The path to the file you want to upload
callback (:obj:`func`, optional): A callback which is passed the number of
bytes uploaded since the last time it was called, used to report progress
Returns:
The requests library response object
"""
extra_headers = extra_headers.copy()
response = None
if os.stat(file.name).st_size == 0:
raise CommError("%s is an empty file" % file.name)
try:
progress = Progress(file, callback=callback)
response = requests.put(url, data=progress, headers=extra_headers)
response.raise_for_status()
except requests.exceptions.RequestException as e:
total = progress.len
status = self._status_request(url, total)
# TODO(adrian): there's probably even more stuff we should add here
# like if we're offline, we should retry then too
if status.status_code in (308, 408, 500, 502, 503, 504):
util.sentry_reraise(retry.TransientException(exc=e))
else:
util.sentry_reraise(e)
return response
def fn(an_arg):
print(an_arg)
try:
if call_num[0] < n:
raise retry.TransientException(
exc=FailException('Failed at call_num: %s' % call_num))
finally:
call_num[0] += 1
return True
def fail():
res = requests.Response()
res.status_code = 401
raise retry.TransientException(exc=requests.HTTPError(response=res))
