def download( self, url, filename, params=None, headers=None, timeout=None, session=None, untar=False, delete_tar=True, extract_path=None, ): """ Download the file from the given url at the current path """ # pylint:disable=too-many-branches logger.debug("Downloading files from url: %s", url) request_headers = self._get_headers(headers=headers) timeout = timeout if timeout is not None else settings.LONG_REQUEST_TIMEOUT session = session or self.session try: response = session.get( url, params=params, headers=request_headers, timeout=timeout, stream=True, ) self.check_response_status(response, url) with open(filename, "wb") as f: # chunk mode response doesn't have content-length so we are # using a custom header here content_length = response.headers.get( "x-polyaxon-content-length") if not content_length: content_length = response.headers.get("content-length") if content_length: for chunk in progress_bar( response.iter_content(chunk_size=1024), expected_size=(int(content_length) / 1024) + 1, ): if chunk: f.write(chunk) else: for chunk in response.iter_content(chunk_size=1024): if chunk: f.write(chunk) if untar: filename = self.untar(filename=filename, delete_tar=delete_tar, extract_path=extract_path) return filename except ( requests.exceptions.ConnectionError, requests.exceptions.RequestException, requests.exceptions.Timeout, requests.exceptions.HTTPError, ) as exception: try: logger.debug("Exception: %s", exception) except TypeError: pass raise PolyaxonShouldExitError( "Error connecting to Polyaxon server on `{}`.\n" "An Error `{}` occurred.\n" "Check your host and ports configuration " "and your internet connection.".format(url, exception))
def request( self, method, url, params=None, data=None, files=None, json=None, # noqa timeout=None, headers=None, session=None, ): """Send a request with the given data as json to the given URL. Args: method: HTTP method url: The url to send the request to. params: Dictionary of values to format url. data: files: json: timeout: headers: extra headers to add to the request. Returns: Request response if successful, Exception otherwise. Raises: PolyaxonHTTPError or one of its subclasses. """ logger.debug("Starting request to url: %s with params: %s, data: %s", url, params, data) request_headers = self._get_headers(headers=headers) timeout = timeout if timeout is not None else settings.LONG_REQUEST_TIMEOUT session = session or self.session try: response = session.request( method, url, params=params, data=data, json=json, headers=request_headers, files=files, timeout=timeout, verify=self.config.verify_ssl, ) except ( requests.exceptions.RequestException, requests.exceptions.Timeout, requests.exceptions.HTTPError, ) as exception: try: logger.debug("Exception: %s", exception, exc_info=True) except TypeError: pass raise PolyaxonShouldExitError( "Error connecting to Polyaxon server on `{}`.\n" "An Error `{}` occurred.\n" "Check your host and ports configuration " "and your internet connection.".format(url, exception)) logger.debug("Response Content: %s, Headers: %s", response.content, response.headers) self.check_response_status(response, url) return response
def upload( self, url, files, files_size, params=None, json_data=None, timeout=None, headers=None, session=None, ): if files_size > settings.WARN_UPLOAD_SIZE: logger.warning( "You are uploading %s, there's a hard limit of %s.\n" "If you have data files in the current directory, " "please make sure to add them to .polyaxonignore or " "add them directly to your data volume, or upload them " "separately using `polyaxon data` command and remove them from here.\n", self.format_sizeof(settings.WARN_UPLOAD_SIZE), self.format_sizeof(settings.MAX_UPLOAD_SIZE), ) if files_size > settings.MAX_UPLOAD_SIZE: raise PolyaxonShouldExitError( "Files too large to sync, please keep it under {}.\n" "If you have data files in the current directory, " "please add them directly to your data volume, or upload them " "separately using `polyaxon data` command and remove them from here.\n" .format(self.format_sizeof(settings.MAX_UPLOAD_SIZE))) files = to_list(files) if json_data: files.append(("json", json.dumps(json_data))) multipart_encoder = MultipartEncoder(fields=files) request_headers = headers or {} request_headers.update( {"Content-Type": multipart_encoder.content_type}) # Attach progress bar progress_callback, callback_bar = self.create_progress_callback( multipart_encoder) multipart_encoder_monitor = MultipartEncoderMonitor( multipart_encoder, progress_callback) timeout = timeout if timeout is not None else settings.LONG_REQUEST_TIMEOUT try: response = self.put( url=url, params=params, data=multipart_encoder_monitor, headers=request_headers, timeout=timeout, session=session, ) finally: # always make sure we clear the console callback_bar.done() return response