def store_file(self, artifact_id, entry, preparer, progress_callback=None):
# write-through cache
cache_path, hit = self._cache.check_md5_obj_path(
entry.digest, entry.size)
if not hit:
shutil.copyfile(entry.local_path, cache_path)
resp = preparer.prepare(lambda: {
"artifactID": artifact_id,
"name": entry.path,
"md5": entry.digest,
})
exists = resp.upload_url is None
if not exists:
with open(entry.local_path, "rb") as file:
# This fails if we don't send the first byte before the signed URL
# expires.
r = self._session.put(
resp.upload_url,
headers={
header.split(":", 1)[0]: header.split(":", 1)[1]
for header in (resp.upload_headers or {})
},
data=Progress(file, callback=progress_callback))
r.raise_for_status()
return exists
def upload_file(self, url, file, callback=None, extra_headers={}):
"""Uploads a file to W&B with failure resumption
Args:
url (str): The url to download
file (str): The path to the file you want to upload
callback (:obj:`func`, optional): A callback which is passed the number of
bytes uploaded since the last time it was called, used to report progress
Returns:
The requests library response object
"""
extra_headers = extra_headers.copy()
response = None
progress = Progress(file, callback=callback)
if progress.len == 0:
raise CommError("%s is an empty file" % file.name)
try:
response = requests.put(
url, data=progress, headers=extra_headers)
response.raise_for_status()
except requests.exceptions.RequestException as e:
status_code = e.response.status_code if e.response != None else 0
# Retry errors from cloud storage or local network issues
if status_code in (308, 409, 429, 500, 502, 503, 504) or isinstance(e, (requests.exceptions.Timeout, requests.exceptions.ConnectionError)):
util.sentry_reraise(retry.TransientException(exc=e))
else:
util.sentry_reraise(e)
return response
def upload_file(self, url, file, callback=None, extra_headers={}):
"""Uploads a file to W&B with failure resumption
Args:
url (str): The url to download
file (str): The path to the file you want to upload
callback (:obj:`func`, optional): A callback which is passed the number of
bytes uploaded since the last time it was called, used to report progress
Returns:
The requests library response object
"""
extra_headers = extra_headers.copy()
response = None
if os.stat(file.name).st_size == 0:
raise CommError("%s is an empty file" % file.name)
try:
progress = Progress(file, callback=callback)
response = requests.put(url, data=progress, headers=extra_headers)
response.raise_for_status()
except requests.exceptions.RequestException as e:
total = progress.len
status = self._status_request(url, total)
# TODO(adrian): there's probably even more stuff we should add here
# like if we're offline, we should retry then too
if status.status_code in (308, 408, 500, 502, 503, 504):
util.sentry_reraise(retry.TransientException(exc=e))
else:
util.sentry_reraise(e)
return response
