def is_temp_failure(self, exc): #IGNORE:W0613
if is_temp_network_error(exc) or isinstance(exc, ssl.SSLError):
# We probably can't use the connection anymore, so use this
# opportunity to reset it
self.conn.reset()
if isinstance(exc, (InternalError, BadDigestError, IncompleteBodyError,
RequestTimeoutError, OperationAbortedError,
SlowDownError, ServiceUnavailableError)):
return True
elif is_temp_network_error(exc):
return True
# In doubt, we retry on 5xx (Server error). However, there are some
# codes where retry is definitely not desired. For 4xx (client error) we
# do not retry in general, but for 408 (Request Timeout) RFC 2616
# specifies that the client may repeat the request without
# modifications.
elif (isinstance(exc, HTTPError)
and ((500 <= exc.status <= 599 and exc.status not in
(501, 505, 508, 510, 511, 523)) or exc.status == 408)):
return True
# Consider all SSL errors as temporary. There are a lot of bug
# reports from people where various SSL errors cause a crash
# but are actually just temporary. On the other hand, we have
# no information if this ever revealed a problem where retrying
# was not the right choice.
elif isinstance(exc, ssl.SSLError):
return True
return False
def is_temp_failure(self, exc): #IGNORE:W0613
if is_temp_network_error(exc) or isinstance(exc, ssl.SSLError):
# We probably can't use the connection anymore, so use this
# opportunity to reset it
self.conn.reset()
if isinstance(exc, (InternalError, BadDigestError, IncompleteBodyError,
RequestTimeoutError, OperationAbortedError,
SlowDownError, ServiceUnavailableError)):
return True
elif is_temp_network_error(exc):
return True
# In doubt, we retry on 5xx (Server error). However, there are some
# codes where retry is definitely not desired. For 4xx (client error) we
# do not retry in general, but for 408 (Request Timeout) RFC 2616
# specifies that the client may repeat the request without
# modifications.
elif (isinstance(exc, HTTPError) and
((500 <= exc.status <= 599
and exc.status not in (501,505,508,510,511,523))
or exc.status == 408)):
return True
# Consider all SSL errors as temporary. There are a lot of bug
# reports from people where various SSL errors cause a crash
# but are actually just temporary. On the other hand, we have
# no information if this ever revealed a problem where retrying
# was not the right choice.
elif isinstance(exc, ssl.SSLError):
return True
return False
def is_temp_failure(self, exc): #IGNORE:W0613
if isinstance(exc, AuthenticationExpired):
return True
# In doubt, we retry on 5xx (Server error). However, there are some
# codes where retry is definitely not desired. For 4xx (client error) we
# do not retry in general, but for 408 (Request Timeout) RFC 2616
# specifies that the client may repeat the request without
# modifications.
elif (isinstance(exc, HTTPError) and
((500 <= exc.status <= 599
and exc.status not in (501,505,508,510,511,523))
or exc.status == 408)):
return True
elif is_temp_network_error(exc):
return True
# Temporary workaround for https://bitbucket.org/nikratio/s3ql/issues/87.
# We still need to find a proper string
elif (isinstance(exc, ssl.SSLError) and
str(exc).startswith('[SSL: BAD_WRITE_RETRY]')):
return True
return False
def is_temp_failure(self, exc): #IGNORE:W0613
if isinstance(exc, AuthenticationExpired):
return True
# In doubt, we retry on 5xx (Server error). However, there are some
# codes where retry is definitely not desired. For 4xx (client error) we
# do not retry in general, but for 408 (Request Timeout) RFC 2616
# specifies that the client may repeat the request without
# modifications. We also retry on 429 (Too Many Requests).
elif (isinstance(exc, HTTPError) and
((500 <= exc.status <= 599
and exc.status not in (501,505,508,510,511,523))
or exc.status in (408,429)
or 'client disconnected' in exc.msg.lower())):
return True
elif is_temp_network_error(exc):
return True
# Temporary workaround for
# https://bitbucket.org/nikratio/s3ql/issues/87 and
# https://bitbucket.org/nikratio/s3ql/issues/252
elif (isinstance(exc, ssl.SSLError) and
(str(exc).startswith('[SSL: BAD_WRITE_RETRY]') or
str(exc).startswith('[SSL: BAD_LENGTH]'))):
return True
return False
def is_temp_failure(self, exc): #IGNORE:W0613
if isinstance(exc, AuthenticationExpired):
return True
# In doubt, we retry on 5xx (Server error). However, there are some
# codes where retry is definitely not desired. For 4xx (client error) we
# do not retry in general, but for 408 (Request Timeout) RFC 2616
# specifies that the client may repeat the request without
# modifications. We also retry on 429 (Too Many Requests).
elif (isinstance(exc, HTTPError) and
((500 <= exc.status <= 599
and exc.status not in (501,505,508,510,511,523))
or exc.status in (408,429)
or 'client disconnected' in exc.msg.lower())):
return True
elif is_temp_network_error(exc):
return True
# Temporary workaround for
# https://bitbucket.org/nikratio/s3ql/issues/87 and
# https://bitbucket.org/nikratio/s3ql/issues/252
elif (isinstance(exc, ssl.SSLError) and
(str(exc).startswith('[SSL: BAD_WRITE_RETRY]') or
str(exc).startswith('[SSL: BAD_LENGTH]'))):
return True
return False
def is_temp_failure(self, exc): #IGNORE:W0613
if isinstance(exc, (InternalError, BadDigestError, IncompleteBodyError,
RequestTimeoutError, OperationAbortedError,
SlowDownError, ServiceUnavailableError)):
return True
elif is_temp_network_error(exc):
return True
# In doubt, we retry on 5xx (Server error). However, there are some
# codes where retry is definitely not desired. For 4xx (client error) we
# do not retry in general, but for 408 (Request Timeout) RFC 2616
# specifies that the client may repeat the request without
# modifications.
elif (isinstance(exc, HTTPError)
and ((500 <= exc.status <= 599 and exc.status not in
(501, 505, 508, 510, 511, 523)) or exc.status == 408)):
return True
# Temporary workaround for
# https://bitbucket.org/nikratio/s3ql/issues/87 and
# https://bitbucket.org/nikratio/s3ql/issues/252
elif (isinstance(exc, ssl.SSLError)
and (str(exc).startswith('[SSL: BAD_WRITE_RETRY]')
or str(exc).startswith('[SSL: BAD_LENGTH]'))):
return True
return False
def _do_request(self, method, path, subres=None, query_string=None,
headers=None, body=None):
'''Send request, read and return response object
This method modifies the *headers* dictionary.
'''
log.debug('started with %r, %r, %r, %r, %r, %r',
method, path, subres, query_string, headers, body)
if headers is None:
headers = CaseInsensitiveDict()
if isinstance(body, (bytes, bytearray, memoryview)):
headers['Content-MD5'] = md5sum_b64(body)
if self.conn is None:
log.debug('no active connection, calling _get_conn()')
self.conn = self._get_conn()
# Construct full path
path = urllib.parse.quote('%s/%s%s' % (self.auth_prefix, self.container_name, path))
if query_string:
s = urllib.parse.urlencode(query_string, doseq=True)
if subres:
path += '?%s&%s' % (subres, s)
else:
path += '?%s' % s
elif subres:
path += '?%s' % subres
headers['X-Auth-Token'] = self.auth_token
try:
resp = self._do_request_inner(method, path, body=body, headers=headers)
except Exception as exc:
if is_temp_network_error(exc):
# We probably can't use the connection anymore
self.conn.disconnect()
raise
# Success
if resp.status >= 200 and resp.status <= 299:
return resp
# Expired auth token
if resp.status == 401:
log.info('OpenStack auth token seems to have expired, requesting new one.')
self.conn.disconnect()
# Force constructing a new connection with a new token, otherwise
# the connection will be reestablished with the same token.
self.conn = None
raise AuthenticationExpired(resp.reason)
# If method == HEAD, server must not return response body
# even in case of errors
self.conn.discard()
if method.upper() == 'HEAD':
raise HTTPError(resp.status, resp.reason, resp.headers)
else:
raise HTTPError(resp.status, resp.reason, resp.headers)
def read(self, size=None):
"""Read up to *size* bytes of object data
For integrity checking to work, this method has to be called until
it returns an empty string, indicating that all data has been read
(and verified).
"""
if size == 0:
return b''
try:
buf = self.conn.read(size)
except Exception as exc:
if is_temp_network_error(exc):
# We probably can't use the connection anymore
self.conn.disconnect()
raise
self.sha1.update(buf)
# Check SHA1 on EOF
# (size == None implies EOF)
if (not buf or size is None) and not self.sha1_checked:
file_hash = self.resp.headers['x-bz-content-sha1'].strip('"')
self.sha1_checked = True
if file_hash != self.sha1.hexdigest():
log.warning('SHA mismatch for %s: %s vs %s', self.key,
file_hash, self.sha1.hexdigest())
raise BadDigestError(
'BadDigest',
'SHA1 header does not agree with calculated SHA1')
return buf
def list(self, prefix='', start_after=''):
log.debug('started with %s, %s', prefix, start_after)
keys_remaining = True
marker = self.prefix + start_after
prefix = self.prefix + prefix
while keys_remaining:
log.debug('requesting with marker=%s', marker)
keys_remaining = None
resp = self._do_request('GET', '/', query_string={ 'prefix': prefix,
'marker': marker,
'max-keys': 1000 })
if not XML_CONTENT_RE.match(resp.headers['Content-Type']):
raise RuntimeError('unexpected content type: %s' %
resp.headers['Content-Type'])
try:
itree = iter(ElementTree.iterparse(self.conn, events=("start", "end")))
(event, root) = next(itree)
root_xmlns_uri = self._tag_xmlns_uri(root)
if root_xmlns_uri is None:
root_xmlns_prefix = ''
else:
# Validate the XML namespace
root_xmlns_prefix = '{%s}' % (root_xmlns_uri, )
if root_xmlns_prefix != self.xml_ns_prefix:
log.error('Unexpected server reply to list operation:\n%s',
self._dump_response(resp, body=None))
raise RuntimeError('List response has %s as root tag, unknown namespace' % root.tag)
for (event, el) in itree:
if event != 'end':
continue
if el.tag == root_xmlns_prefix + 'IsTruncated':
keys_remaining = (el.text == 'true')
elif el.tag == root_xmlns_prefix + 'Contents':
marker = el.findtext(root_xmlns_prefix + 'Key')
yield marker[len(self.prefix):]
root.clear()
except Exception as exc:
if is_temp_network_error(exc):
# We probably can't use the connection anymore
self.conn.disconnect()
raise
except GeneratorExit:
# Need to read rest of response
self.conn.discard()
break
if keys_remaining is None:
raise RuntimeError('Could not parse body')
def _do_request(self, method, path, subres=None, query_string=None,
headers=None, body=None):
'''Send request, read and return response object
This method modifies the *headers* dictionary.
'''
log.debug('started with %r, %r, %r, %r, %r, %r',
method, path, subres, query_string, headers, body)
if headers is None:
headers = CaseInsensitiveDict()
if isinstance(body, (bytes, bytearray, memoryview)):
headers['Content-MD5'] = md5sum_b64(body)
if self.conn is None:
log.debug('no active connection, calling _get_conn()')
self.conn = self._get_conn()
# Construct full path
path = urllib.parse.quote('%s/%s%s' % (self.auth_prefix, self.container_name, path))
if query_string:
s = urllib.parse.urlencode(query_string, doseq=True)
if subres:
path += '?%s&%s' % (subres, s)
else:
path += '?%s' % s
elif subres:
path += '?%s' % subres
headers['X-Auth-Token'] = self.auth_token
try:
resp = self._do_request_inner(method, path, body=body, headers=headers)
except Exception as exc:
if is_temp_network_error(exc) or isinstance(exc, ssl.SSLError):
# We probably can't use the connection anymore
self.conn.disconnect()
raise
# Success
if resp.status >= 200 and resp.status <= 299:
return resp
# Expired auth token
if resp.status == 401:
self._do_authentication_expired(resp.reason)
# raises AuthenticationExpired
# If method == HEAD, server must not return response body
# even in case of errors
self.conn.discard()
if method.upper() == 'HEAD':
raise HTTPError(resp.status, resp.reason, resp.headers)
else:
raise HTTPError(resp.status, resp.reason, resp.headers)
def is_temp_failure(self, exc):
if is_temp_network_error(exc) or isinstance(exc, ssl.SSLError):
# We better reset our connections
self._reset_connections()
if is_temp_network_error(exc):
return True
elif (isinstance(exc, B2Error)
and (exc.code == 'bad_auth_token'
or exc.code == 'expired_auth_token')):
self._reset_authorization_values()
return True
elif (isinstance(exc, B2Error)
and (exc.code == 'cap_exceeded'
or exc.code == 'test_mode_cap_exceeded')
and self.retry_on_cap_exceeded):
return True
elif isinstance(exc, HTTPError) and exc.status == 401:
self._reset_authorization_values()
return True
elif (isinstance(exc, HTTPError)
and ((500 <= exc.status <= 599) or # server errors
exc.status == 408 or # request timeout
exc.status == 429)): # too many requests
return True
# Consider all SSL errors as temporary. There are a lot of bug
# reports from people where various SSL errors cause a crash
# but are actually just temporary. On the other hand, we have
# no information if this ever revealed a problem where retrying
# was not the right choice.
elif isinstance(exc, ssl.SSLError):
return True
return False
def is_temp_failure(self, exc): #IGNORE:W0613
if is_temp_network_error(exc) or isinstance(exc, ssl.SSLError):
# We probably can't use the connection anymore, so use this
# opportunity to reset it
self.conn.reset()
return True
elif isinstance(exc, RequestError) and (
500 <= exc.code <= 599 or exc.code == 408):
return True
# Not clear at all what is happening here, but in doubt we retry
elif isinstance(exc, ServerResponseError):
return True
return False
def is_temp_failure(self, exc):
log.warning("Got exception %s: %s" %
(type(exc).__name__, str(exc)))
if isinstance(exc, (MD5Error, SizeError)):
return True
elif is_temp_network_error(exc):
return True
elif (isinstance(exc, RequestError) and
((500 <= exc.status_code <= 599 and exc.status_code not in
(501, 505, 508, 510, 511, 523)) or exc.status_code
in (400, 401, 408, 429, RequestError.CODE.CONN_EXCEPTION,
RequestError.CODE.FAILED_SUBREQUEST,
RequestError.CODE.INCOMPLETE_RESULT,
RequestError.CODE.REFRESH_FAILED))):
return True
return False
def is_temp_failure(self, exc): #IGNORE:W0613
if isinstance(exc, AuthenticationExpired):
return True
# In doubt, we retry on 5xx (Server error). However, there are some
# codes where retry is definitely not desired. For 4xx (client error) we
# do not retry in general, but for 408 (Request Timeout) RFC 2616
# specifies that the client may repeat the request without
# modifications.
elif (isinstance(exc, HTTPError)
and ((500 <= exc.status <= 599 and exc.status not in
(501, 505, 508, 510, 511, 523)) or exc.status == 408)):
return True
elif is_temp_network_error(exc):
return True
return False
def is_temp_failure(self, exc): #IGNORE:W0613
if isinstance(exc, AuthenticationExpired):
return True
# In doubt, we retry on 5xx (Server error). However, there are some
# codes where retry is definitely not desired. For 4xx (client error) we
# do not retry in general, but for 408 (Request Timeout) RFC 2616
# specifies that the client may repeat the request without
# modifications.
elif (isinstance(exc, HTTPError) and
((500 <= exc.status <= 599
and exc.status not in (501,505,508,510,511,523))
or exc.status == 408)):
return True
elif is_temp_network_error(exc):
return True
return False
def is_temp_failure(self, exc): #IGNORE:W0613
if isinstance(exc, (InternalError, BadDigestError, IncompleteBodyError,
RequestTimeoutError, OperationAbortedError,
SlowDownError, ServiceUnavailableError)):
return True
elif is_temp_network_error(exc):
return True
# In doubt, we retry on 5xx (Server error). However, there are some
# codes where retry is definitely not desired. For 4xx (client error) we
# do not retry in general, but for 408 (Request Timeout) RFC 2616
# specifies that the client may repeat the request without
# modifications.
elif (isinstance(exc, HTTPError)
and ((500 <= exc.status <= 599 and exc.status not in
(501, 505, 508, 510, 511, 523)) or exc.status == 408)):
return True
return False
def is_temp_failure(self, exc):
if isinstance(exc, AuthenticationExpired):
return True
if isinstance(exc, ssl.SSLError):
return True
if is_temp_network_error(exc):
return True
if isinstance(exc, HTTPError):
return True
if isinstance(exc, B2Error):
if (exc.status == 400 or \
exc.status == 408 or \
(exc.status >= 500 and exc.status <= 599)):
return True
return False
def is_temp_failure(self, exc): #IGNORE:W0613
if isinstance(exc, (InternalError, BadDigestError, IncompleteBodyError,
RequestTimeoutError, OperationAbortedError,
SlowDownError, ServiceUnavailableError)):
return True
elif is_temp_network_error(exc):
return True
# In doubt, we retry on 5xx (Server error). However, there are some
# codes where retry is definitely not desired. For 4xx (client error) we
# do not retry in general, but for 408 (Request Timeout) RFC 2616
# specifies that the client may repeat the request without
# modifications.
elif (isinstance(exc, HTTPError) and
((500 <= exc.status <= 599
and exc.status not in (501,505,508,510,511,523))
or exc.status == 408)):
return True
return False
def _do_upload_request(self, headers=None, body=None):
upload_url_info = self._get_upload_url_info()
headers['Authorization'] = upload_url_info['authorizationToken']
if self.test_mode_fail_some_uploads:
headers['X-Bz-Test-Mode'] = 'fail_some_uploads'
upload_url_info['isUploading'] = True
try:
response, response_body = self._do_request(
upload_url_info['connection'], 'POST', upload_url_info['path'],
headers, body)
except B2Error as exc:
if exc.status == 503:
# storage url too busy, change it
self._invalidate_upload_url(upload_url_info)
raise
except (ConnectionClosed, ConnectionTimedOut):
# storage url too busy, change it
self._invalidate_upload_url(upload_url_info)
raise
except Exception as exc:
if is_temp_network_error(exc) or isinstance(exc, ssl.SSLError):
# we better get a new upload url
self._invalidate_upload_url(upload_url_info)
else:
upload_url_info['connection'].reset()
upload_url_info['isUploading'] = False
raise
upload_url_info['isUploading'] = False
json_response = json.loads(response_body.decode('utf-8'))
return json_response
def is_temp_failure(self, exc): #IGNORE:W0613
if is_temp_network_error(exc) or isinstance(exc, ssl.SSLError):
# We probably can't use the connection anymore, so use this
# opportunity to reset it
self.conn.reset()
return True
elif isinstance(exc, RequestError) and (500 <= exc.code <= 599
or exc.code == 408):
return True
elif isinstance(exc, AccessTokenExpired):
del self.access_token[self.refresh_token]
return True
# Not clear at all what is happening here, but in doubt we retry
elif isinstance(exc, ServerResponseError):
return True
if g_auth and isinstance(exc, g_auth.exceptions.TransportError):
return True
return False
def is_temp_failure(self, exc): #IGNORE:W0613
if isinstance(exc, AuthenticationExpired):
return True
# In doubt, we retry on 5xx (Server error). However, there are some
# codes where retry is definitely not desired. For 4xx (client error) we
# do not retry in general, but for 408 (Request Timeout) RFC 2616
# specifies that the client may repeat the request without
# modifications.
elif (isinstance(exc, HTTPError)
and ((500 <= exc.status <= 599 and exc.status not in
(501, 505, 508, 510, 511, 523)) or exc.status == 408)):
return True
elif is_temp_network_error(exc):
return True
# Temporary workaround for https://bitbucket.org/nikratio/s3ql/issues/87.
# We still need to find a proper string
elif (isinstance(exc, ssl.SSLError)
and str(exc).startswith('[SSL: BAD_WRITE_RETRY]')):
return True
return False
def _send_request(self, method, path, headers, subres=None, query_string=None, body=None):
'''Add authentication and send request
Returns the response object.
'''
if not isinstance(headers, CaseInsensitiveDict):
headers = CaseInsensitiveDict(headers)
self._authorize_request(method, path, headers, subres)
# Construct full path
if not self.hostname.startswith(self.bucket_name):
path = '/%s%s' % (self.bucket_name, path)
path = urllib.parse.quote(path)
if query_string:
s = urllib.parse.urlencode(query_string, doseq=True)
if subres:
path += '?%s&%s' % (subres, s)
else:
path += '?%s' % s
elif subres:
path += '?%s' % subres
# We can probably remove the assertions at some point and
# call self.conn.read_response() directly
def read_response():
resp = self.conn.read_response()
assert resp.method == method
assert resp.path == path
return resp
use_expect_100c = not self.options.get('disable-expect100', False)
try:
log.debug('sending %s %s', method, path)
if body is None or isinstance(body, (bytes, bytearray, memoryview)):
self.conn.send_request(method, path, body=body, headers=headers)
else:
body_len = os.fstat(body.fileno()).st_size
self.conn.send_request(method, path, expect100=use_expect_100c,
headers=headers, body=BodyFollowing(body_len))
if use_expect_100c:
resp = read_response()
if resp.status != 100: # Error
return resp
try:
copyfileobj(body, self.conn, BUFSIZE)
except ConnectionClosed:
# Server closed connection while we were writing body data -
# but we may still be able to read an error response
try:
resp = read_response()
except ConnectionClosed: # No server response available
pass
else:
if resp.status >= 400: # Got error response
return resp
log.warning('Server broke connection during upload, but signaled '
'%d %s', resp.status, resp.reason)
# Re-raise first ConnectionClosed exception
raise
return read_response()
except Exception as exc:
if is_temp_network_error(exc):
# We probably can't use the connection anymore
self.conn.disconnect()
raise
def _do_request(self,
method,
path,
conn,
headers=None,
body=None,
auth_token=None,
download_body=True,
body_size=None):
"""Send request, read and return response object
This method modifies the *headers* dictionary.
conn must by a HTTPConnection
When download_body is True, need to receive data before making new connection
"""
def _debug_body(b):
if isinstance(b, str):
return b
elif b is None:
return "None"
else:
return 'byte_body'
def _debug_hostname(c):
try:
return c.hostname
except:
return "None"
log.debug('started with %r, %r, %r, %r, %r', method,
_debug_hostname(conn), path, headers, _debug_body(body))
if headers is None:
headers = CaseInsensitiveDict()
if auth_token is None:
headers['Authorization'] = self.auth_token
else:
headers['Authorization'] = auth_token
if self.test_string:
headers['X-Bz-Test-Mode'] = self.test_string
try:
if isinstance(body, io.FileIO):
if body_size is None:
raise ValueError(
"Body size is necessary when uploading from file")
conn.send_request(method,
path,
headers=headers,
body=BodyFollowing(body_size))
while True:
buf = body.read(BUFSIZE)
if not buf:
break
conn.write(buf)
else:
conn.send_request(method, path, headers=headers, body=body)
resp = conn.read_response()
if download_body or resp.status != 200:
body = conn.readall()
else:
# caller need to download body itself before making new request
body = None
except Exception as exc:
if is_temp_network_error(exc):
# We probably can't use the connection anymore
conn.disconnect()
raise
if resp.status == 200 or resp.status == 206:
return resp, body
try:
# error code is in body
j = json.loads(str(body, encoding='UTF-8'))
except ValueError:
raise HTTPError(resp.status, resp.reason, resp.headers)
# Expired auth token
if resp.status == 401:
if j['code'] == 'expired_auth_token':
log.info(
'BackBlaze auth token seems to have expired, requesting new one.'
)
self.conn_api.disconnect()
self.conn_download.disconnect()
# Force constructing a new connection with a new token, otherwise
# the connection will be reestablished with the same token.
self.conn_api = None
self.conn_download = None
self._login()
raise AuthenticationExpired(j['message'])
else:
raise AuthorizationError(j['message'])
# File not found
if resp.status == 404:
raise NoSuchObject(path)
# Backend error
raise B2Error(j['status'], j['code'], j['message'], headers=headers)
def _send_request(self,
method,
path,
headers,
subres=None,
query_string=None,
body=None):
'''Add authentication and send request
Returns the response object.
'''
if not isinstance(headers, CaseInsensitiveDict):
headers = CaseInsensitiveDict(headers)
self._authorize_request(method, path, headers, subres)
# Construct full path
if not self.hostname.startswith(self.bucket_name):
path = '/%s%s' % (self.bucket_name, path)
path = urllib.parse.quote(path)
if query_string:
s = urllib.parse.urlencode(query_string, doseq=True)
if subres:
path += '?%s&%s' % (subres, s)
else:
path += '?%s' % s
elif subres:
path += '?%s' % subres
# We can probably remove the assertions at some point and
# call self.conn.read_response() directly
def read_response():
resp = self.conn.read_response()
assert resp.method == method
assert resp.path == path
return resp
use_expect_100c = not self.options.get('disable-expect100', False)
try:
log.debug('sending %s %s', method, path)
if body is None or isinstance(body,
(bytes, bytearray, memoryview)):
self.conn.send_request(method,
path,
body=body,
headers=headers)
else:
body_len = os.fstat(body.fileno()).st_size
self.conn.send_request(method,
path,
expect100=use_expect_100c,
headers=headers,
body=BodyFollowing(body_len))
if use_expect_100c:
resp = read_response()
if resp.status != 100: # Error
return resp
try:
copyfileobj(body, self.conn, BUFSIZE)
except ConnectionClosed:
# Server closed connection while we were writing body data -
# but we may still be able to read an error response
try:
resp = read_response()
except ConnectionClosed: # No server response available
pass
else:
if resp.status >= 400: # Got error response
return resp
log.warning(
'Server broke connection during upload, but signaled '
'%d %s', resp.status, resp.reason)
# Re-raise first ConnectionClosed exception
raise
return read_response()
except Exception as exc:
if is_temp_network_error(exc):
# We probably can't use the connection anymore
self.conn.disconnect()
raise
def list(self, prefix='', start_after=''):
log.debug('started with %s, %s', prefix, start_after)
keys_remaining = True
marker = self.prefix + start_after
prefix = self.prefix + prefix
while keys_remaining:
log.debug('requesting with marker=%s', marker)
keys_remaining = None
resp = self._do_request('GET',
'/',
query_string={
'prefix': prefix,
'marker': marker,
'max-keys': 1000
})
if not XML_CONTENT_RE.match(resp.headers['Content-Type']):
raise RuntimeError('unexpected content type: %s' %
resp.headers['Content-Type'])
try:
itree = iter(
ElementTree.iterparse(self.conn, events=("start", "end")))
(event, root) = next(itree)
root_xmlns_uri = self._tag_xmlns_uri(root)
if root_xmlns_uri is None:
root_xmlns_prefix = ''
else:
# Validate the XML namespace
root_xmlns_prefix = '{%s}' % (root_xmlns_uri, )
if root_xmlns_prefix != self.xml_ns_prefix:
log.error(
'Unexpected server reply to list operation:\n%s',
self._dump_response(resp, body=None))
raise RuntimeError(
'List response has %s as root tag, unknown namespace'
% root.tag)
for (event, el) in itree:
if event != 'end':
continue
if el.tag == root_xmlns_prefix + 'IsTruncated':
keys_remaining = (el.text == 'true')
elif el.tag == root_xmlns_prefix + 'Contents':
marker = el.findtext(root_xmlns_prefix + 'Key')
yield marker[len(self.prefix):]
root.clear()
except Exception as exc:
if is_temp_network_error(exc):
# We probably can't use the connection anymore
self.conn.disconnect()
raise
except GeneratorExit:
# Need to read rest of response
self.conn.discard()
break
if keys_remaining is None:
raise RuntimeError('Could not parse body')
