def download(
self,
url,
filename,
params=None,
headers=None,
timeout=None,
session=None,
untar=False,
delete_tar=True,
extract_path=None,
):
"""
Download the file from the given url at the current path
"""
# pylint:disable=too-many-branches
logger.debug("Downloading files from url: %s", url)
request_headers = self._get_headers(headers=headers)
timeout = timeout if timeout is not None else settings.LONG_REQUEST_TIMEOUT
session = session or self.session
try:
response = session.get(
url,
params=params,
headers=request_headers,
timeout=timeout,
stream=True,
)
self.check_response_status(response, url)
with open(filename, "wb") as f:
# chunk mode response doesn't have content-length so we are
# using a custom header here
content_length = response.headers.get(
"x-polyaxon-content-length")
if not content_length:
content_length = response.headers.get("content-length")
if content_length:
for chunk in progress_bar(
response.iter_content(chunk_size=1024),
expected_size=(int(content_length) / 1024) + 1,
):
if chunk:
f.write(chunk)
else:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
if untar:
filename = self.untar(filename=filename,
delete_tar=delete_tar,
extract_path=extract_path)
return filename
except (
requests.exceptions.ConnectionError,
requests.exceptions.RequestException,
requests.exceptions.Timeout,
requests.exceptions.HTTPError,
) as exception:
try:
logger.debug("Exception: %s", exception)
except TypeError:
pass
raise PolyaxonShouldExitError(
"Error connecting to Polyaxon server on `{}`.\n"
"An Error `{}` occurred.\n"
"Check your host and ports configuration "
"and your internet connection.".format(url, exception))
def request(
self,
method,
url,
params=None,
data=None,
files=None,
json=None, # noqa
timeout=None,
headers=None,
session=None,
):
"""Send a request with the given data as json to the given URL.
Args:
method: HTTP method
url: The url to send the request to.
params: Dictionary of values to format url.
data:
files:
json:
timeout:
headers: extra headers to add to the request.
Returns:
Request response if successful, Exception otherwise.
Raises:
PolyaxonHTTPError or one of its subclasses.
"""
logger.debug("Starting request to url: %s with params: %s, data: %s",
url, params, data)
request_headers = self._get_headers(headers=headers)
timeout = timeout if timeout is not None else settings.LONG_REQUEST_TIMEOUT
session = session or self.session
try:
response = session.request(
method,
url,
params=params,
data=data,
json=json,
headers=request_headers,
files=files,
timeout=timeout,
verify=self.config.verify_ssl,
)
except (
requests.exceptions.RequestException,
requests.exceptions.Timeout,
requests.exceptions.HTTPError,
) as exception:
try:
logger.debug("Exception: %s", exception, exc_info=True)
except TypeError:
pass
raise PolyaxonShouldExitError(
"Error connecting to Polyaxon server on `{}`.\n"
"An Error `{}` occurred.\n"
"Check your host and ports configuration "
"and your internet connection.".format(url, exception))
logger.debug("Response Content: %s, Headers: %s", response.content,
response.headers)
self.check_response_status(response, url)
return response
def upload(
self,
url,
files,
files_size,
params=None,
json_data=None,
timeout=None,
headers=None,
session=None,
):
if files_size > settings.WARN_UPLOAD_SIZE:
logger.warning(
"You are uploading %s, there's a hard limit of %s.\n"
"If you have data files in the current directory, "
"please make sure to add them to .polyaxonignore or "
"add them directly to your data volume, or upload them "
"separately using `polyaxon data` command and remove them from here.\n",
self.format_sizeof(settings.WARN_UPLOAD_SIZE),
self.format_sizeof(settings.MAX_UPLOAD_SIZE),
)
if files_size > settings.MAX_UPLOAD_SIZE:
raise PolyaxonShouldExitError(
"Files too large to sync, please keep it under {}.\n"
"If you have data files in the current directory, "
"please add them directly to your data volume, or upload them "
"separately using `polyaxon data` command and remove them from here.\n"
.format(self.format_sizeof(settings.MAX_UPLOAD_SIZE)))
files = to_list(files)
if json_data:
files.append(("json", json.dumps(json_data)))
multipart_encoder = MultipartEncoder(fields=files)
request_headers = headers or {}
request_headers.update(
{"Content-Type": multipart_encoder.content_type})
# Attach progress bar
progress_callback, callback_bar = self.create_progress_callback(
multipart_encoder)
multipart_encoder_monitor = MultipartEncoderMonitor(
multipart_encoder, progress_callback)
timeout = timeout if timeout is not None else settings.LONG_REQUEST_TIMEOUT
try:
response = self.put(
url=url,
params=params,
data=multipart_encoder_monitor,
headers=request_headers,
timeout=timeout,
session=session,
)
finally:
# always make sure we clear the console
callback_bar.done()
return response
