def upload(
self,
url,
files,
files_size,
params=None,
json_data=None,
timeout=None,
headers=None,
session=None,
):
upload_size_warn = os.environ.get("POLYAXON_UPLOAD_SIZE_MAX", 1024 * 1024 * 50)
upload_size_max = os.environ.get("POLYAXON_UPLOAD_SIZE_MAX", 1024 * 1024 * 500)
if files_size > upload_size_warn:
logger.warning(
"You are uploading %s, there's a hard limit of %s.\n"
"If you have data files in the current directory, "
"please make sure to add them to .polyaxonignore or "
"add them directly to your data volume, or upload them "
"separately using `polyaxon data` command and remove them from here.\n",
format_sizeof(files_size),
format_sizeof(upload_size_max),
)
if files_size > upload_size_max:
raise PolyaxonShouldExitError(
"Files too large to sync, please keep it under {}.\n"
"If you have data files in the current directory, "
"please add them directly to your data volume, or upload them "
"separately using `polyaxon data` command and remove them from here.\n".format(
format_sizeof(upload_size_max)
)
)
files = to_list(files)
if json_data:
files.append(("json", json.dumps(json_data)))
multipart_encoder = MultipartEncoder(fields=files)
request_headers = self._get_headers(headers=headers)
request_headers.update({"Content-Type": multipart_encoder.content_type})
# Attach progress bar
progress_callback, callback_bar = create_progress_callback(multipart_encoder)
multipart_encoder_monitor = MultipartEncoderMonitor(
multipart_encoder, progress_callback
)
timeout = timeout if timeout is not None else settings.LONG_REQUEST_TIMEOUT
session = session or requests.Session()
try:
response = session.post(
url=url,
params=params,
data=multipart_encoder_monitor,
headers=request_headers,
timeout=timeout,
)
finally:
# always make sure we clear the console
callback_bar.done()
return response
def test_format_sizeof(self):
assert format_sizeof(10) == "10.0B"
assert format_sizeof(10000) == "9.8KiB"
assert format_sizeof(100000) == "97.7KiB"
assert format_sizeof(10000000) == "9.5MiB"
assert format_sizeof(10000000000) == "9.3GiB"