def build_admin_request(conn, method, resource = '', headers=None, data='',
query_args=None, params=None):
path = conn.calling_format.build_path_base('admin', resource)
auth_path = conn.calling_format.build_auth_path('admin', resource)
host = conn.calling_format.build_host(conn.server_name(), 'admin')
if query_args:
path += '?' + query_args
boto.log.debug('path=%s' % path)
auth_path += '?' + query_args
boto.log.debug('auth_path=%s' % auth_path)
return AWSAuthConnection.build_base_http_request(conn, method, path,
auth_path, params, headers, data, host)
def _build_request(conn, method, basepath='', resource = '', headers=None,
data=None, special_first_param=None, params=None):
path = conn.calling_format.build_path_base(basepath, resource)
auth_path = conn.calling_format.build_auth_path(basepath, resource)
host = conn.calling_format.build_host(conn.server_name(), '')
if special_first_param:
path += '?' + special_first_param
boto.log.debug('path=%s' % path)
auth_path += '?' + special_first_param
boto.log.debug('auth_path=%s' % auth_path)
return AWSAuthConnection.build_base_http_request(
conn, method, path, auth_path, params, headers, data, host)
def test_build_base_http_request_noproxy(self):
os.environ['no_proxy'] = 'mockservice.cc-zone-1.amazonaws.com'
conn = AWSAuthConnection('mockservice.cc-zone-1.amazonaws.com',
aws_access_key_id='access_key',
aws_secret_access_key='secret',
suppress_consec_slashes=False,
proxy="127.0.0.1",
proxy_user="******",
proxy_pass="******",
proxy_port="8180")
request = conn.build_base_http_request('GET', '/', None)
del os.environ['no_proxy']
self.assertEqual(request.path, '/')
def test_build_base_http_request_noproxy(self):
os.environ['no_proxy'] = 'mockservice.cc-zone-1.amazonaws.com'
conn = AWSAuthConnection(
'mockservice.cc-zone-1.amazonaws.com',
aws_access_key_id='access_key',
aws_secret_access_key='secret',
suppress_consec_slashes=False,
proxy="127.0.0.1",
proxy_user="******",
proxy_pass="******",
proxy_port="8180"
)
request = conn.build_base_http_request('GET', '/', None)
del os.environ['no_proxy']
self.assertEqual(request.path, '/')
def build_admin_request(conn, method, resource = '', headers=None, data='',
query_args=None, params=None):
"""
Build an administative request adapted from the build_request()
method of boto.connection
"""
path = conn.calling_format.build_path_base('admin', resource)
auth_path = conn.calling_format.build_auth_path('admin', resource)
host = conn.calling_format.build_host(conn.server_name(), 'admin')
if query_args:
path += '?' + query_args
boto.log.debug('path=%s' % path)
auth_path += '?' + query_args
boto.log.debug('auth_path=%s' % auth_path)
return AWSAuthConnection.build_base_http_request(conn, method, path,
auth_path, params, headers, data, host)
def base_http_request(conn, method, basepath='', resource='', headers=None,
data=None, special_first_param=None, params=None):
"""
Returns a ``AWSAuthConnection.build_base_http_request`` call with the
preserving of the special params done by ``build``.
"""
# request meta data
md = build(
conn,
method,
basepath=basepath,
resource=resource,
headers=headers,
data=data,
special_first_param=special_first_param,
params=params,
)
return AWSAuthConnection.build_base_http_request(
md.conn, md.method, md.path,
md.auth_path, md.params, md.headers,
md.data, md.host)
def _upload_file_bytes(self, conn, http_conn, fp, file_length,
total_bytes_uploaded, cb, num_cb, md5sum):
"""
Makes one attempt to upload file bytes, using an existing resumable
upload connection.
Returns etag from server upon success.
Raises ResumableUploadException if any problems occur.
"""
buf = fp.read(self.BUFFER_SIZE)
if cb:
# The cb_count represents the number of full buffers to send between
# cb executions.
if num_cb > 2:
cb_count = file_length / self.BUFFER_SIZE / (num_cb - 2)
elif num_cb < 0:
cb_count = -1
else:
cb_count = 0
i = 0
cb(total_bytes_uploaded, file_length)
# Build resumable upload headers for the transfer. Don't send a
# Content-Range header if the file is 0 bytes long, because the
# resumable upload protocol uses an *inclusive* end-range (so, sending
# 'bytes 0-0/1' would actually mean you're sending a 1-byte file).
put_headers = {}
if file_length:
if total_bytes_uploaded == file_length:
range_header = self._build_content_range_header(
'*', file_length)
else:
range_header = self._build_content_range_header(
'%d-%d' % (total_bytes_uploaded, file_length - 1),
file_length)
put_headers['Content-Range'] = range_header
# Set Content-Length to the total bytes we'll send with this PUT.
put_headers['Content-Length'] = str(file_length - total_bytes_uploaded)
http_request = AWSAuthConnection.build_base_http_request(
conn,
'PUT',
path=self.tracker_uri_path,
auth_path=None,
headers=put_headers,
host=self.tracker_uri_host)
http_conn.putrequest('PUT', http_request.path)
for k in put_headers:
http_conn.putheader(k, put_headers[k])
http_conn.endheaders()
# Turn off debug on http connection so upload content isn't included
# in debug stream.
http_conn.set_debuglevel(0)
while buf:
http_conn.send(buf)
md5sum.update(buf)
total_bytes_uploaded += len(buf)
if cb:
i += 1
if i == cb_count or cb_count == -1:
cb(total_bytes_uploaded, file_length)
i = 0
buf = fp.read(self.BUFFER_SIZE)
if cb:
cb(total_bytes_uploaded, file_length)
if total_bytes_uploaded != file_length:
# Abort (and delete the tracker file) so if the user retries
# they'll start a new resumable upload rather than potentially
# attempting to pick back up later where we left off.
raise ResumableUploadException(
'File changed during upload: EOF at %d bytes of %d byte file.'
% (total_bytes_uploaded, file_length),
ResumableTransferDisposition.ABORT)
resp = http_conn.getresponse()
body = resp.read()
# Restore http connection debug level.
http_conn.set_debuglevel(conn.debug)
if resp.status == 200:
return resp.getheader('etag') # Success
# Retry timeout (408) and status 500 and 503 errors after a delay.
elif resp.status in [408, 500, 503]:
disposition = ResumableTransferDisposition.WAIT_BEFORE_RETRY
else:
# Catch all for any other error codes.
disposition = ResumableTransferDisposition.ABORT
raise ResumableUploadException(
'Got response code %d while attempting '
'upload (%s)' % (resp.status, resp.reason), disposition)
def _upload_file_bytes(self, conn, http_conn, fp, file_length,
total_bytes_uploaded, cb, num_cb):
"""
Makes one attempt to upload file bytes, using an existing resumable
upload connection.
Returns etag from server upon success.
Raises ResumableUploadException if any problems occur.
"""
buf = fp.read(self.BUFFER_SIZE)
if cb:
if num_cb > 2:
cb_count = file_length / self.BUFFER_SIZE / (num_cb-2)
elif num_cb < 0:
cb_count = -1
else:
cb_count = 0
i = 0
cb(total_bytes_uploaded, file_length)
# Build resumable upload headers for the transfer. Don't send a
# Content-Range header if the file is 0 bytes long, because the
# resumable upload protocol uses an *inclusive* end-range (so, sending
# 'bytes 0-0/1' would actually mean you're sending a 1-byte file).
put_headers = {}
if file_length:
range_header = self._build_content_range_header(
'%d-%d' % (total_bytes_uploaded, file_length - 1),
file_length)
put_headers['Content-Range'] = range_header
# Set Content-Length to the total bytes we'll send with this PUT.
put_headers['Content-Length'] = str(file_length - total_bytes_uploaded)
http_request = AWSAuthConnection.build_base_http_request(
conn, 'PUT', path=self.tracker_uri_path, auth_path=None,
headers=put_headers, host=self.tracker_uri_host)
http_conn.putrequest('PUT', http_request.path)
for k in put_headers:
http_conn.putheader(k, put_headers[k])
http_conn.endheaders()
# Turn off debug on http connection so upload content isn't included
# in debug stream.
http_conn.set_debuglevel(0)
while buf:
http_conn.send(buf)
total_bytes_uploaded += len(buf)
if cb:
i += 1
if i == cb_count or cb_count == -1:
cb(total_bytes_uploaded, file_length)
i = 0
buf = fp.read(self.BUFFER_SIZE)
if cb:
cb(total_bytes_uploaded, file_length)
if total_bytes_uploaded != file_length:
# Abort (and delete the tracker file) so if the user retries
# they'll start a new resumable upload rather than potentially
# attempting to pick back up later where we left off.
raise ResumableUploadException(
'File changed during upload: EOF at %d bytes of %d byte file.' %
(total_bytes_uploaded, file_length),
ResumableTransferDisposition.ABORT)
resp = http_conn.getresponse()
body = resp.read()
# Restore http connection debug level.
http_conn.set_debuglevel(conn.debug)
if resp.status == 200:
return resp.getheader('etag') # Success
# Retry timeout (408) and status 500 and 503 errors after a delay.
elif resp.status in [408, 500, 503]:
disposition = ResumableTransferDisposition.WAIT_BEFORE_RETRY
else:
# Catch all for any other error codes.
disposition = ResumableTransferDisposition.ABORT
raise ResumableUploadException('Got response code %d while attempting '
'upload (%s)' %
(resp.status, resp.reason), disposition)
def _upload_file_bytes(self, conn, http_conn, fp, file_length,
total_bytes_uploaded, cb, num_cb):
"""
Makes one attempt to upload file bytes, using an existing resumable
upload connection.
Returns etag from server upon success.
Raises ResumableUploadException if any problems occur.
"""
buf = fp.read(self.BUFFER_SIZE)
if cb:
if num_cb > 2:
cb_count = file_length / self.BUFFER_SIZE / (num_cb-2)
elif num_cb < 0:
cb_count = -1
else:
cb_count = 0
i = 0
cb(total_bytes_uploaded, file_length)
# Build resumable upload headers for the transfer. Don't send a
# Content-Range header if the file is 0 bytes long, because the
# resumable upload protocol uses an *inclusive* end-range (so, sending
# 'bytes 0-0/1' would actually mean you're sending a 1-byte file).
put_headers = {}
if file_length:
range_header = self._build_content_range_header(
'%d-%d' % (total_bytes_uploaded, file_length - 1),
file_length)
put_headers['Content-Range'] = range_header
# Set Content-Length to the total bytes we'll send with this PUT.
put_headers['Content-Length'] = str(file_length - total_bytes_uploaded)
http_request = AWSAuthConnection.build_base_http_request(
conn, 'PUT', path=self.tracker_uri_path, auth_path=None,
headers=put_headers, host=self.tracker_uri_host)
http_conn.putrequest('PUT', http_request.path)
for k in put_headers:
http_conn.putheader(k, put_headers[k])
http_conn.endheaders()
# Turn off debug on http connection so upload content isn't included
# in debug stream.
http_conn.set_debuglevel(0)
while buf:
http_conn.send(buf)
total_bytes_uploaded += len(buf)
if cb:
i += 1
if i == cb_count or cb_count == -1:
cb(total_bytes_uploaded, file_length)
i = 0
buf = fp.read(self.BUFFER_SIZE)
if cb:
cb(total_bytes_uploaded, file_length)
if total_bytes_uploaded != file_length:
# Abort (and delete the tracker file) so if the user retries
# they'll start a new resumable uplaod rather than potentially
# attempting to pick back up later where we left off.
raise ResumableUploadException(
'File changed during upload: EOF at %d bytes of %d byte file.' %
(total_bytes_uploaded, file_length),
ResumableTransferDisposition.ABORT)
resp = http_conn.getresponse()
body = resp.read()
# Restore http connection debug level.
http_conn.set_debuglevel(conn.debug)
additional_note = ''
if resp.status == 200:
return resp.getheader('etag') # Success
elif resp.status == 408:
# Request Timeout. Try again later within the current process.
disposition = ResumableTransferDisposition.WAIT_BEFORE_RETRY
elif resp.status/100 == 4:
# Abort for any other 4xx errors.
disposition = ResumableTransferDisposition.ABORT
# Add some more informative note for particular 4xx error codes.
if resp.status == 400:
additional_note = ('This can happen for various reasons; one '
'common case is if you attempt to upload a '
'different size file on a already partially '
'uploaded resumable upload')
# Retry status 500 and 503 errors after a delay.
elif resp.status == 500 or resp.status == 503:
disposition = ResumableTransferDisposition.WAIT_BEFORE_RETRY
else:
# Catch all for any other error codes.
disposition = ResumableTransferDisposition.ABORT
raise ResumableUploadException('Got response code %d while attempting '
'upload (%s)%s' %
(resp.status, resp.reason,
additional_note), disposition)
def _UploadFileBytes(self, conn, http_conn, fp, file_length,
total_bytes_uploaded, cb, num_cb, headers):
"""Attempts to upload file bytes.
Makes a single attempt using an existing resumable upload connection.
Args:
conn: HTTPConnection from the boto Key.
http_conn: Separate HTTPConnection for the transfer.
fp: File pointer containing bytes to upload.
file_length: Total length of the file.
total_bytes_uploaded: The total number of bytes uploaded.
cb: Progress callback function that takes (progress, total_size).
num_cb: Granularity of the callback (maximum number of times the
callback will be called during the file transfer). If negative,
perform callback with each buffer read.
headers: Headers to be used in the upload requests.
Returns:
(etag, generation, metageneration) from service upon success.
Raises:
ResumableUploadException if any problems occur.
"""
buf = fp.read(self.BUFFER_SIZE)
if cb:
# The cb_count represents the number of full buffers to send between
# cb executions.
if num_cb > 2:
cb_count = file_length / self.BUFFER_SIZE / (num_cb - 2)
elif num_cb < 0:
cb_count = -1
else:
cb_count = 0
i = 0
cb(total_bytes_uploaded, file_length)
# Build resumable upload headers for the transfer. Don't send a
# Content-Range header if the file is 0 bytes long, because the
# resumable upload protocol uses an *inclusive* end-range (so, sending
# 'bytes 0-0/1' would actually mean you're sending a 1-byte file).
put_headers = headers.copy() if headers else {}
if file_length:
if total_bytes_uploaded == file_length:
range_header = self._BuildContentRangeHeader('*', file_length)
else:
range_header = self._BuildContentRangeHeader(
'%d-%d' % (total_bytes_uploaded, file_length - 1), file_length)
put_headers['Content-Range'] = range_header
# Set Content-Length to the total bytes we'll send with this PUT.
put_headers['Content-Length'] = str(file_length - total_bytes_uploaded)
http_request = AWSAuthConnection.build_base_http_request(
conn,
'PUT',
path=self.upload_url_path,
auth_path=None,
headers=put_headers,
host=self.upload_url_host)
http_conn.putrequest('PUT', http_request.path)
for k in put_headers:
http_conn.putheader(k, put_headers[k])
http_conn.endheaders()
# Turn off debug on http connection so upload content isn't included
# in debug stream.
http_conn.set_debuglevel(0)
while buf:
# Some code is duplicated here, but separating the PY2 and PY3 paths makes
# this easier to remove PY2 blocks when we move to PY3 only.
if six.PY2:
http_conn.send(buf)
total_bytes_uploaded += len(buf)
else:
if isinstance(buf, bytes):
http_conn.send(buf)
total_bytes_uploaded += len(buf)
else:
# Probably a unicode/str object, try encoding.
buf_bytes = buf.encode(UTF8)
http_conn.send(buf_bytes)
total_bytes_uploaded += len(buf_bytes)
if cb:
i += 1
if i == cb_count or cb_count == -1:
cb(total_bytes_uploaded, file_length)
i = 0
buf = fp.read(self.BUFFER_SIZE)
# Restore http connection debug level.
http_conn.set_debuglevel(conn.debug)
if cb:
cb(total_bytes_uploaded, file_length)
if total_bytes_uploaded != file_length:
# Abort (and delete the tracker file) so if the user retries
# they'll start a new resumable upload rather than potentially
# attempting to pick back up later where we left off.
raise ResumableUploadException(
'File changed during upload: EOF at %d bytes of %d byte file.' %
(total_bytes_uploaded, file_length),
ResumableTransferDisposition.ABORT)
resp = http_conn.getresponse()
if resp.status == 200:
# Success.
return (resp.getheader('etag'), resp.getheader('x-goog-generation'),
resp.getheader('x-goog-metageneration'))
# Retry timeout (408) and status 429, 500 and 503 errors after a delay.
elif resp.status in [408, 429, 500, 503]:
disposition = ResumableTransferDisposition.WAIT_BEFORE_RETRY
else:
# Catch all for any other error codes.
disposition = ResumableTransferDisposition.ABORT
raise ResumableUploadException(
'Got response code %d while attempting '
'upload (%s)' % (resp.status, resp.reason), disposition)
def _upload_file_bytes(self, conn, http_conn, fp, file_length,
total_bytes_uploaded, cb, num_cb):
"""
Makes one attempt to upload file bytes, using an existing resumable
upload connection.
Returns etag from server upon success.
Raises ResumableUploadException if any problems occur.
"""
buf = fp.read(self.BUFFER_SIZE)
if cb:
if num_cb > 2:
cb_count = file_length / self.BUFFER_SIZE / (num_cb-2)
elif num_cb < 0:
cb_count = -1
else:
cb_count = 0
i = 0
cb(total_bytes_uploaded, file_length)
# Build resumable upload headers for the transfer. Don't send a
# Content-Range header if the file is 0 bytes long, because the
# resumable upload protocol uses an *inclusive* end-range (so, sending
# 'bytes 0-0/1' would actually mean you're sending a 1-byte file).
put_headers = {}
if file_length:
range_header = self._build_content_range_header(
'%d-%d' % (total_bytes_uploaded, file_length - 1),
file_length)
put_headers['Content-Range'] = range_header
# Set Content-Length to the total bytes we'll send with this PUT.
put_headers['Content-Length'] = str(file_length - total_bytes_uploaded)
http_request = AWSAuthConnection.build_base_http_request(
conn, 'PUT', path=self.tracker_uri_path, auth_path=None,
headers=put_headers, host=self.tracker_uri_host)
http_conn.putrequest('PUT', http_request.path)
for k in put_headers:
http_conn.putheader(k, put_headers[k])
http_conn.endheaders()
# Turn off debug on http connection so upload content isn't included
# in debug stream.
http_conn.set_debuglevel(0)
while buf:
try:
http_conn.send(buf)
except socket.error, e:
# The server will close the connection if you send more
# bytes in a resumed upload than the original file size.
if (e.args[0] == errno.EPIPE and
total_bytes_uploaded != file_length):
raise ResumableUploadException(
'File changed during upload: sent %d bytes for file for '
'which original size was %d bytes file.' %
(total_bytes_uploaded, file_length),
ResumableTransferDisposition.ABORT)
# Some other failure happened. Pass the exception on.
raise e
total_bytes_uploaded += len(buf)
if cb:
i += 1
if i == cb_count or cb_count == -1:
cb(total_bytes_uploaded, file_length)
i = 0
buf = fp.read(self.BUFFER_SIZE)
def _UploadFileBytes(self, conn, http_conn, fp, file_length,
total_bytes_uploaded, cb, num_cb, headers):
"""Attempts to upload file bytes.
Makes a single attempt using an existing resumable upload connection.
Args:
conn: HTTPConnection from the boto Key.
http_conn: Separate HTTPConnection for the transfer.
fp: File pointer containing bytes to upload.
file_length: Total length of the file.
total_bytes_uploaded: The total number of bytes uploaded.
cb: Progress callback function that takes (progress, total_size).
num_cb: Granularity of the callback (maximum number of times the
callback will be called during the file transfer). If negative,
perform callback with each buffer read.
headers: Headers to be used in the upload requests.
Returns:
(etag, generation, metageneration) from service upon success.
Raises:
ResumableUploadException if any problems occur.
"""
buf = fp.read(self.BUFFER_SIZE)
if cb:
# The cb_count represents the number of full buffers to send between
# cb executions.
if num_cb > 2:
cb_count = file_length / self.BUFFER_SIZE / (num_cb - 2)
elif num_cb < 0:
cb_count = -1
else:
cb_count = 0
i = 0
cb(total_bytes_uploaded, file_length)
# Build resumable upload headers for the transfer. Don't send a
# Content-Range header if the file is 0 bytes long, because the
# resumable upload protocol uses an *inclusive* end-range (so, sending
# 'bytes 0-0/1' would actually mean you're sending a 1-byte file).
put_headers = headers.copy() if headers else {}
if file_length:
if total_bytes_uploaded == file_length:
range_header = self._BuildContentRangeHeader('*', file_length)
else:
range_header = self._BuildContentRangeHeader(
'%d-%d' % (total_bytes_uploaded, file_length - 1),
file_length)
put_headers['Content-Range'] = range_header
# Set Content-Length to the total bytes we'll send with this PUT.
put_headers['Content-Length'] = str(file_length - total_bytes_uploaded)
http_request = AWSAuthConnection.build_base_http_request(
conn,
'PUT',
path=self.upload_url_path,
auth_path=None,
headers=put_headers,
host=self.upload_url_host)
http_conn.putrequest('PUT', http_request.path)
for k in put_headers:
http_conn.putheader(k, put_headers[k])
http_conn.endheaders()
# Turn off debug on http connection so upload content isn't included
# in debug stream.
http_conn.set_debuglevel(0)
while buf:
# Some code is duplicated here, but separating the PY2 and PY3 paths makes
# this easier to remove PY2 blocks when we move to PY3 only.
if six.PY2:
http_conn.send(buf)
total_bytes_uploaded += len(buf)
else:
if isinstance(buf, bytes):
http_conn.send(buf)
total_bytes_uploaded += len(buf)
else:
# Probably a unicode/str object, try encoding.
buf_bytes = buf.encode(UTF8)
http_conn.send(buf_bytes)
total_bytes_uploaded += len(buf_bytes)
if cb:
i += 1
if i == cb_count or cb_count == -1:
cb(total_bytes_uploaded, file_length)
i = 0
buf = fp.read(self.BUFFER_SIZE)
# Restore http connection debug level.
http_conn.set_debuglevel(conn.debug)
if cb:
cb(total_bytes_uploaded, file_length)
if total_bytes_uploaded != file_length:
# Abort (and delete the tracker file) so if the user retries
# they'll start a new resumable upload rather than potentially
# attempting to pick back up later where we left off.
raise ResumableUploadException(
'File changed during upload: EOF at %d bytes of %d byte file.'
% (total_bytes_uploaded, file_length),
ResumableTransferDisposition.ABORT)
resp = http_conn.getresponse()
if resp.status == 200:
# Success.
return (resp.getheader('etag'),
resp.getheader('x-goog-generation'),
resp.getheader('x-goog-metageneration'))
# Retry timeout (408) and status 429, 500 and 503 errors after a delay.
elif resp.status in [408, 429, 500, 503]:
disposition = ResumableTransferDisposition.WAIT_BEFORE_RETRY
else:
# Catch all for any other error codes.
disposition = ResumableTransferDisposition.ABORT
raise ResumableUploadException(
'Got response code %d while attempting '
'upload (%s)' % (resp.status, resp.reason), disposition)