def build_admin_request(conn, method, resource = '', headers=None, data='', query_args=None, params=None): path = conn.calling_format.build_path_base('admin', resource) auth_path = conn.calling_format.build_auth_path('admin', resource) host = conn.calling_format.build_host(conn.server_name(), 'admin') if query_args: path += '?' + query_args boto.log.debug('path=%s' % path) auth_path += '?' + query_args boto.log.debug('auth_path=%s' % auth_path) return AWSAuthConnection.build_base_http_request(conn, method, path, auth_path, params, headers, data, host)
def _build_request(conn, method, basepath='', resource = '', headers=None, data=None, special_first_param=None, params=None): path = conn.calling_format.build_path_base(basepath, resource) auth_path = conn.calling_format.build_auth_path(basepath, resource) host = conn.calling_format.build_host(conn.server_name(), '') if special_first_param: path += '?' + special_first_param boto.log.debug('path=%s' % path) auth_path += '?' + special_first_param boto.log.debug('auth_path=%s' % auth_path) return AWSAuthConnection.build_base_http_request( conn, method, path, auth_path, params, headers, data, host)
def test_build_base_http_request_noproxy(self): os.environ['no_proxy'] = 'mockservice.cc-zone-1.amazonaws.com' conn = AWSAuthConnection('mockservice.cc-zone-1.amazonaws.com', aws_access_key_id='access_key', aws_secret_access_key='secret', suppress_consec_slashes=False, proxy="127.0.0.1", proxy_user="******", proxy_pass="******", proxy_port="8180") request = conn.build_base_http_request('GET', '/', None) del os.environ['no_proxy'] self.assertEqual(request.path, '/')
def test_build_base_http_request_noproxy(self): os.environ['no_proxy'] = 'mockservice.cc-zone-1.amazonaws.com' conn = AWSAuthConnection( 'mockservice.cc-zone-1.amazonaws.com', aws_access_key_id='access_key', aws_secret_access_key='secret', suppress_consec_slashes=False, proxy="127.0.0.1", proxy_user="******", proxy_pass="******", proxy_port="8180" ) request = conn.build_base_http_request('GET', '/', None) del os.environ['no_proxy'] self.assertEqual(request.path, '/')
def build_admin_request(conn, method, resource = '', headers=None, data='', query_args=None, params=None): """ Build an administative request adapted from the build_request() method of boto.connection """ path = conn.calling_format.build_path_base('admin', resource) auth_path = conn.calling_format.build_auth_path('admin', resource) host = conn.calling_format.build_host(conn.server_name(), 'admin') if query_args: path += '?' + query_args boto.log.debug('path=%s' % path) auth_path += '?' + query_args boto.log.debug('auth_path=%s' % auth_path) return AWSAuthConnection.build_base_http_request(conn, method, path, auth_path, params, headers, data, host)
def base_http_request(conn, method, basepath='', resource='', headers=None, data=None, special_first_param=None, params=None): """ Returns a ``AWSAuthConnection.build_base_http_request`` call with the preserving of the special params done by ``build``. """ # request meta data md = build( conn, method, basepath=basepath, resource=resource, headers=headers, data=data, special_first_param=special_first_param, params=params, ) return AWSAuthConnection.build_base_http_request( md.conn, md.method, md.path, md.auth_path, md.params, md.headers, md.data, md.host)
def _upload_file_bytes(self, conn, http_conn, fp, file_length, total_bytes_uploaded, cb, num_cb, md5sum): """ Makes one attempt to upload file bytes, using an existing resumable upload connection. Returns etag from server upon success. Raises ResumableUploadException if any problems occur. """ buf = fp.read(self.BUFFER_SIZE) if cb: # The cb_count represents the number of full buffers to send between # cb executions. if num_cb > 2: cb_count = file_length / self.BUFFER_SIZE / (num_cb - 2) elif num_cb < 0: cb_count = -1 else: cb_count = 0 i = 0 cb(total_bytes_uploaded, file_length) # Build resumable upload headers for the transfer. Don't send a # Content-Range header if the file is 0 bytes long, because the # resumable upload protocol uses an *inclusive* end-range (so, sending # 'bytes 0-0/1' would actually mean you're sending a 1-byte file). put_headers = {} if file_length: if total_bytes_uploaded == file_length: range_header = self._build_content_range_header( '*', file_length) else: range_header = self._build_content_range_header( '%d-%d' % (total_bytes_uploaded, file_length - 1), file_length) put_headers['Content-Range'] = range_header # Set Content-Length to the total bytes we'll send with this PUT. put_headers['Content-Length'] = str(file_length - total_bytes_uploaded) http_request = AWSAuthConnection.build_base_http_request( conn, 'PUT', path=self.tracker_uri_path, auth_path=None, headers=put_headers, host=self.tracker_uri_host) http_conn.putrequest('PUT', http_request.path) for k in put_headers: http_conn.putheader(k, put_headers[k]) http_conn.endheaders() # Turn off debug on http connection so upload content isn't included # in debug stream. http_conn.set_debuglevel(0) while buf: http_conn.send(buf) md5sum.update(buf) total_bytes_uploaded += len(buf) if cb: i += 1 if i == cb_count or cb_count == -1: cb(total_bytes_uploaded, file_length) i = 0 buf = fp.read(self.BUFFER_SIZE) if cb: cb(total_bytes_uploaded, file_length) if total_bytes_uploaded != file_length: # Abort (and delete the tracker file) so if the user retries # they'll start a new resumable upload rather than potentially # attempting to pick back up later where we left off. raise ResumableUploadException( 'File changed during upload: EOF at %d bytes of %d byte file.' % (total_bytes_uploaded, file_length), ResumableTransferDisposition.ABORT) resp = http_conn.getresponse() body = resp.read() # Restore http connection debug level. http_conn.set_debuglevel(conn.debug) if resp.status == 200: return resp.getheader('etag') # Success # Retry timeout (408) and status 500 and 503 errors after a delay. elif resp.status in [408, 500, 503]: disposition = ResumableTransferDisposition.WAIT_BEFORE_RETRY else: # Catch all for any other error codes. disposition = ResumableTransferDisposition.ABORT raise ResumableUploadException( 'Got response code %d while attempting ' 'upload (%s)' % (resp.status, resp.reason), disposition)
def _upload_file_bytes(self, conn, http_conn, fp, file_length, total_bytes_uploaded, cb, num_cb): """ Makes one attempt to upload file bytes, using an existing resumable upload connection. Returns etag from server upon success. Raises ResumableUploadException if any problems occur. """ buf = fp.read(self.BUFFER_SIZE) if cb: if num_cb > 2: cb_count = file_length / self.BUFFER_SIZE / (num_cb-2) elif num_cb < 0: cb_count = -1 else: cb_count = 0 i = 0 cb(total_bytes_uploaded, file_length) # Build resumable upload headers for the transfer. Don't send a # Content-Range header if the file is 0 bytes long, because the # resumable upload protocol uses an *inclusive* end-range (so, sending # 'bytes 0-0/1' would actually mean you're sending a 1-byte file). put_headers = {} if file_length: range_header = self._build_content_range_header( '%d-%d' % (total_bytes_uploaded, file_length - 1), file_length) put_headers['Content-Range'] = range_header # Set Content-Length to the total bytes we'll send with this PUT. put_headers['Content-Length'] = str(file_length - total_bytes_uploaded) http_request = AWSAuthConnection.build_base_http_request( conn, 'PUT', path=self.tracker_uri_path, auth_path=None, headers=put_headers, host=self.tracker_uri_host) http_conn.putrequest('PUT', http_request.path) for k in put_headers: http_conn.putheader(k, put_headers[k]) http_conn.endheaders() # Turn off debug on http connection so upload content isn't included # in debug stream. http_conn.set_debuglevel(0) while buf: http_conn.send(buf) total_bytes_uploaded += len(buf) if cb: i += 1 if i == cb_count or cb_count == -1: cb(total_bytes_uploaded, file_length) i = 0 buf = fp.read(self.BUFFER_SIZE) if cb: cb(total_bytes_uploaded, file_length) if total_bytes_uploaded != file_length: # Abort (and delete the tracker file) so if the user retries # they'll start a new resumable upload rather than potentially # attempting to pick back up later where we left off. raise ResumableUploadException( 'File changed during upload: EOF at %d bytes of %d byte file.' % (total_bytes_uploaded, file_length), ResumableTransferDisposition.ABORT) resp = http_conn.getresponse() body = resp.read() # Restore http connection debug level. http_conn.set_debuglevel(conn.debug) if resp.status == 200: return resp.getheader('etag') # Success # Retry timeout (408) and status 500 and 503 errors after a delay. elif resp.status in [408, 500, 503]: disposition = ResumableTransferDisposition.WAIT_BEFORE_RETRY else: # Catch all for any other error codes. disposition = ResumableTransferDisposition.ABORT raise ResumableUploadException('Got response code %d while attempting ' 'upload (%s)' % (resp.status, resp.reason), disposition)
def _upload_file_bytes(self, conn, http_conn, fp, file_length, total_bytes_uploaded, cb, num_cb): """ Makes one attempt to upload file bytes, using an existing resumable upload connection. Returns etag from server upon success. Raises ResumableUploadException if any problems occur. """ buf = fp.read(self.BUFFER_SIZE) if cb: if num_cb > 2: cb_count = file_length / self.BUFFER_SIZE / (num_cb-2) elif num_cb < 0: cb_count = -1 else: cb_count = 0 i = 0 cb(total_bytes_uploaded, file_length) # Build resumable upload headers for the transfer. Don't send a # Content-Range header if the file is 0 bytes long, because the # resumable upload protocol uses an *inclusive* end-range (so, sending # 'bytes 0-0/1' would actually mean you're sending a 1-byte file). put_headers = {} if file_length: range_header = self._build_content_range_header( '%d-%d' % (total_bytes_uploaded, file_length - 1), file_length) put_headers['Content-Range'] = range_header # Set Content-Length to the total bytes we'll send with this PUT. put_headers['Content-Length'] = str(file_length - total_bytes_uploaded) http_request = AWSAuthConnection.build_base_http_request( conn, 'PUT', path=self.tracker_uri_path, auth_path=None, headers=put_headers, host=self.tracker_uri_host) http_conn.putrequest('PUT', http_request.path) for k in put_headers: http_conn.putheader(k, put_headers[k]) http_conn.endheaders() # Turn off debug on http connection so upload content isn't included # in debug stream. http_conn.set_debuglevel(0) while buf: http_conn.send(buf) total_bytes_uploaded += len(buf) if cb: i += 1 if i == cb_count or cb_count == -1: cb(total_bytes_uploaded, file_length) i = 0 buf = fp.read(self.BUFFER_SIZE) if cb: cb(total_bytes_uploaded, file_length) if total_bytes_uploaded != file_length: # Abort (and delete the tracker file) so if the user retries # they'll start a new resumable uplaod rather than potentially # attempting to pick back up later where we left off. raise ResumableUploadException( 'File changed during upload: EOF at %d bytes of %d byte file.' % (total_bytes_uploaded, file_length), ResumableTransferDisposition.ABORT) resp = http_conn.getresponse() body = resp.read() # Restore http connection debug level. http_conn.set_debuglevel(conn.debug) additional_note = '' if resp.status == 200: return resp.getheader('etag') # Success elif resp.status == 408: # Request Timeout. Try again later within the current process. disposition = ResumableTransferDisposition.WAIT_BEFORE_RETRY elif resp.status/100 == 4: # Abort for any other 4xx errors. disposition = ResumableTransferDisposition.ABORT # Add some more informative note for particular 4xx error codes. if resp.status == 400: additional_note = ('This can happen for various reasons; one ' 'common case is if you attempt to upload a ' 'different size file on a already partially ' 'uploaded resumable upload') # Retry status 500 and 503 errors after a delay. elif resp.status == 500 or resp.status == 503: disposition = ResumableTransferDisposition.WAIT_BEFORE_RETRY else: # Catch all for any other error codes. disposition = ResumableTransferDisposition.ABORT raise ResumableUploadException('Got response code %d while attempting ' 'upload (%s)%s' % (resp.status, resp.reason, additional_note), disposition)
def _UploadFileBytes(self, conn, http_conn, fp, file_length, total_bytes_uploaded, cb, num_cb, headers): """Attempts to upload file bytes. Makes a single attempt using an existing resumable upload connection. Args: conn: HTTPConnection from the boto Key. http_conn: Separate HTTPConnection for the transfer. fp: File pointer containing bytes to upload. file_length: Total length of the file. total_bytes_uploaded: The total number of bytes uploaded. cb: Progress callback function that takes (progress, total_size). num_cb: Granularity of the callback (maximum number of times the callback will be called during the file transfer). If negative, perform callback with each buffer read. headers: Headers to be used in the upload requests. Returns: (etag, generation, metageneration) from service upon success. Raises: ResumableUploadException if any problems occur. """ buf = fp.read(self.BUFFER_SIZE) if cb: # The cb_count represents the number of full buffers to send between # cb executions. if num_cb > 2: cb_count = file_length / self.BUFFER_SIZE / (num_cb - 2) elif num_cb < 0: cb_count = -1 else: cb_count = 0 i = 0 cb(total_bytes_uploaded, file_length) # Build resumable upload headers for the transfer. Don't send a # Content-Range header if the file is 0 bytes long, because the # resumable upload protocol uses an *inclusive* end-range (so, sending # 'bytes 0-0/1' would actually mean you're sending a 1-byte file). put_headers = headers.copy() if headers else {} if file_length: if total_bytes_uploaded == file_length: range_header = self._BuildContentRangeHeader('*', file_length) else: range_header = self._BuildContentRangeHeader( '%d-%d' % (total_bytes_uploaded, file_length - 1), file_length) put_headers['Content-Range'] = range_header # Set Content-Length to the total bytes we'll send with this PUT. put_headers['Content-Length'] = str(file_length - total_bytes_uploaded) http_request = AWSAuthConnection.build_base_http_request( conn, 'PUT', path=self.upload_url_path, auth_path=None, headers=put_headers, host=self.upload_url_host) http_conn.putrequest('PUT', http_request.path) for k in put_headers: http_conn.putheader(k, put_headers[k]) http_conn.endheaders() # Turn off debug on http connection so upload content isn't included # in debug stream. http_conn.set_debuglevel(0) while buf: # Some code is duplicated here, but separating the PY2 and PY3 paths makes # this easier to remove PY2 blocks when we move to PY3 only. if six.PY2: http_conn.send(buf) total_bytes_uploaded += len(buf) else: if isinstance(buf, bytes): http_conn.send(buf) total_bytes_uploaded += len(buf) else: # Probably a unicode/str object, try encoding. buf_bytes = buf.encode(UTF8) http_conn.send(buf_bytes) total_bytes_uploaded += len(buf_bytes) if cb: i += 1 if i == cb_count or cb_count == -1: cb(total_bytes_uploaded, file_length) i = 0 buf = fp.read(self.BUFFER_SIZE) # Restore http connection debug level. http_conn.set_debuglevel(conn.debug) if cb: cb(total_bytes_uploaded, file_length) if total_bytes_uploaded != file_length: # Abort (and delete the tracker file) so if the user retries # they'll start a new resumable upload rather than potentially # attempting to pick back up later where we left off. raise ResumableUploadException( 'File changed during upload: EOF at %d bytes of %d byte file.' % (total_bytes_uploaded, file_length), ResumableTransferDisposition.ABORT) resp = http_conn.getresponse() if resp.status == 200: # Success. return (resp.getheader('etag'), resp.getheader('x-goog-generation'), resp.getheader('x-goog-metageneration')) # Retry timeout (408) and status 429, 500 and 503 errors after a delay. elif resp.status in [408, 429, 500, 503]: disposition = ResumableTransferDisposition.WAIT_BEFORE_RETRY else: # Catch all for any other error codes. disposition = ResumableTransferDisposition.ABORT raise ResumableUploadException( 'Got response code %d while attempting ' 'upload (%s)' % (resp.status, resp.reason), disposition)
def _upload_file_bytes(self, conn, http_conn, fp, file_length, total_bytes_uploaded, cb, num_cb): """ Makes one attempt to upload file bytes, using an existing resumable upload connection. Returns etag from server upon success. Raises ResumableUploadException if any problems occur. """ buf = fp.read(self.BUFFER_SIZE) if cb: if num_cb > 2: cb_count = file_length / self.BUFFER_SIZE / (num_cb-2) elif num_cb < 0: cb_count = -1 else: cb_count = 0 i = 0 cb(total_bytes_uploaded, file_length) # Build resumable upload headers for the transfer. Don't send a # Content-Range header if the file is 0 bytes long, because the # resumable upload protocol uses an *inclusive* end-range (so, sending # 'bytes 0-0/1' would actually mean you're sending a 1-byte file). put_headers = {} if file_length: range_header = self._build_content_range_header( '%d-%d' % (total_bytes_uploaded, file_length - 1), file_length) put_headers['Content-Range'] = range_header # Set Content-Length to the total bytes we'll send with this PUT. put_headers['Content-Length'] = str(file_length - total_bytes_uploaded) http_request = AWSAuthConnection.build_base_http_request( conn, 'PUT', path=self.tracker_uri_path, auth_path=None, headers=put_headers, host=self.tracker_uri_host) http_conn.putrequest('PUT', http_request.path) for k in put_headers: http_conn.putheader(k, put_headers[k]) http_conn.endheaders() # Turn off debug on http connection so upload content isn't included # in debug stream. http_conn.set_debuglevel(0) while buf: try: http_conn.send(buf) except socket.error, e: # The server will close the connection if you send more # bytes in a resumed upload than the original file size. if (e.args[0] == errno.EPIPE and total_bytes_uploaded != file_length): raise ResumableUploadException( 'File changed during upload: sent %d bytes for file for ' 'which original size was %d bytes file.' % (total_bytes_uploaded, file_length), ResumableTransferDisposition.ABORT) # Some other failure happened. Pass the exception on. raise e total_bytes_uploaded += len(buf) if cb: i += 1 if i == cb_count or cb_count == -1: cb(total_bytes_uploaded, file_length) i = 0 buf = fp.read(self.BUFFER_SIZE)