def test_put_separate(conn):
data = DUMMY_DATA
conn.send_request('PUT', '/allgood', body=BodyFollowing(len(data)))
conn.write(data)
resp = conn.read_response()
conn.discard()
assert resp.status == 204
assert resp.length == 0
assert resp.reason == 'Ok, but no MD5'
headers = CaseInsensitiveDict()
headers['Content-MD5'] = b64encode(
hashlib.md5(data).digest()).decode('ascii')
conn.send_request('PUT',
'/allgood',
body=BodyFollowing(len(data)),
headers=headers)
conn.write(data)
resp = conn.read_response()
conn.discard()
assert resp.status == 204
assert resp.length == 0
assert resp.reason == 'MD5 matched'
headers['Content-MD5'] = 'nUzaJEag3tOdobQVU/39GA=='
conn.send_request('PUT',
'/allgood',
body=BodyFollowing(len(data)),
headers=headers)
conn.write(data)
resp = conn.read_response()
conn.discard()
assert resp.status == 400
assert resp.reason.startswith('MD5 mismatch')
def test_aborted_write1(conn, monkeypatch, random_fh):
BUFSIZE = 64 * 1024
# monkeypatch request handler
def do_PUT(self):
# Read half the data, then generate error and
# close connection
self.rfile.read(BUFSIZE)
self.send_error(code=401, message='Please stop!')
self.close_connection = True
monkeypatch.setattr(MockRequestHandler, 'do_PUT', do_PUT)
# Send request
conn.send_request('PUT',
'/big_object',
body=BodyFollowing(BUFSIZE * 50),
expect100=True)
resp = conn.read_response()
assert resp.status == 100
assert resp.length == 0
# Try to write data
with pytest.raises(ConnectionClosed):
for _ in range(50):
conn.write(random_fh.read(BUFSIZE))
# Nevertheless, try to read response
resp = conn.read_response()
assert resp.status == 401
assert resp.reason == 'Please stop!'
def test_100cont_2(conn, monkeypatch):
def handle_expect_100(self):
self.send_error(403)
monkeypatch.setattr(MockRequestHandler, 'handle_expect_100',
handle_expect_100)
conn.send_request('PUT',
'/fail_with_403',
body=BodyFollowing(256),
expect100=True)
with pytest.raises(dugong.StateError):
conn.send_request('PUT',
'/fail_with_403',
body=BodyFollowing(256),
expect100=True)
conn.read_response()
conn.readall()
def test_write_toolittle3(conn):
conn.send_request('GET', '/send_10_bytes')
conn.send_request('PUT', '/allgood', body=BodyFollowing(42))
conn.write(DUMMY_DATA[:24])
resp = conn.read_response()
assert resp.status == 200
assert resp.path == '/send_10_bytes'
assert len(conn.readall()) == 10
with pytest.raises(dugong.StateError):
conn.read_response()
def test_100cont(conn, monkeypatch):
path = '/check_this_out'
def handle_expect_100(self):
if self.path != path:
self.send_error(500,
'Assertion error, %s != %s' % (self.path, path))
else:
self.send_error(403)
monkeypatch.setattr(MockRequestHandler, 'handle_expect_100',
handle_expect_100)
conn.send_request('PUT', path, body=BodyFollowing(256), expect100=True)
resp = conn.read_response()
assert resp.status == 403
conn.discard()
def handle_expect_100(self):
if self.path != path:
self.send_error(500,
'Assertion error, %s != %s' % (self.path, path))
return
self.send_response_only(100)
self.end_headers()
return True
monkeypatch.setattr(MockRequestHandler, 'handle_expect_100',
handle_expect_100)
conn.send_request('PUT', path, body=BodyFollowing(256), expect100=True)
resp = conn.read_response()
assert resp.status == 100
assert resp.length == 0
conn.write(DUMMY_DATA[:256])
resp = conn.read_response()
assert resp.status == 204
assert resp.length == 0
def _do_request_inner(self, method, path, body, headers):
'''The guts of the _do_request method'''
log.debug('started with %s %s', method, path)
use_expect_100c = not self.options.get('disable-expect100', False)
if body is None or isinstance(body, (bytes, bytearray, memoryview)):
self.conn.send_request(method, path, body=body, headers=headers)
return self.conn.read_response()
body_len = os.fstat(body.fileno()).st_size
self.conn.send_request(method,
path,
expect100=use_expect_100c,
headers=headers,
body=BodyFollowing(body_len))
if use_expect_100c:
log.debug('waiting for 100-continue')
resp = self.conn.read_response()
if resp.status != 100:
return resp
log.debug('writing body data')
try:
shutil.copyfileobj(body, self.conn, BUFSIZE)
except ConnectionClosed:
log.debug('interrupted write, server closed connection')
# Server closed connection while we were writing body data -
# but we may still be able to read an error response
try:
resp = self.conn.read_response()
except ConnectionClosed: # No server response available
log.debug('no response available in buffer')
pass
else:
if resp.status >= 400: # error response
return resp
log.warning(
'Server broke connection during upload, but signaled '
'%d %s', resp.status, resp.reason)
# Re-raise original error
raise
return self.conn.read_response()
def test_send_timeout(conn, monkeypatch, random_fh):
conn.timeout = 1
def do_PUT(self):
# Read just a tiny bit
self.rfile.read(256)
# We need to sleep, or the rest of the incoming data will
# be parsed as the next request.
time.sleep(2 * conn.timeout)
monkeypatch.setattr(MockRequestHandler, 'do_PUT', do_PUT)
# We don't know how much data can be buffered, so we
# claim to send a lot and do so in a loop.
len_ = 1024**3
conn.send_request('PUT', '/recv_something', body=BodyFollowing(len_))
with pytest.raises(dugong.ConnectionTimedOut):
while len_ > 0:
conn.write(random_fh.read(min(len_, 16 * 1024)))
def write_fh(self,
fh,
key: str,
md5: bytes,
metadata: Optional[Dict[str, Any]] = None,
size: Optional[int] = None):
'''Write data from byte stream *fh* into *key*.
*fh* must be seekable. If *size* is None, *fh* must also implement
`fh.fileno()` so that the size can be determined through `os.fstat`.
*md5* must be the (binary) md5 checksum of the data.
'''
metadata = json.dumps({
'metadata':
_wrap_user_meta(metadata if metadata else {}),
'md5Hash':
b64encode(md5).decode(),
'name':
self.prefix + key,
})
# Google Storage uses Content-Length to read the object data, so we
# don't have to worry about the boundary occurring in the object data.
boundary = 'foo_bar_baz'
headers = CaseInsensitiveDict()
headers['Content-Type'] = 'multipart/related; boundary=%s' % boundary
body_prefix = '\n'.join(
('--' + boundary, 'Content-Type: application/json; charset=UTF-8',
'', metadata, '--' + boundary,
'Content-Type: application/octet-stream', '', '')).encode()
body_suffix = ('\n--%s--\n' % boundary).encode()
body_size = len(body_prefix) + len(body_suffix)
if size is not None:
body_size += size
else:
body_size += os.fstat(fh.fileno()).st_size
path = '/upload/storage/v1/b/%s/o' % (urllib.parse.quote(
self.bucket_name, safe=''), )
query_string = {'uploadType': 'multipart'}
try:
resp = self._do_request('POST',
path,
query_string=query_string,
headers=headers,
body=BodyFollowing(body_size))
except RequestError as exc:
exc = _map_request_error(exc, key)
if exc:
raise exc
raise
assert resp.status == 100
fh.seek(0)
md5_run = hashlib.md5()
try:
self.conn.write(body_prefix)
while True:
buf = fh.read(BUFSIZE)
if not buf:
break
self.conn.write(buf)
md5_run.update(buf)
self.conn.write(body_suffix)
except ConnectionClosed:
# Server closed connection while we were writing body data -
# but we may still be able to read an error response
try:
resp = self.conn.read_response()
except ConnectionClosed: # No server response available
pass
else:
log.warning(
'Server broke connection during upload, signaled '
'%d %s', resp.status, resp.reason)
# Re-raise first ConnectionClosed exception
raise
if md5_run.digest() != md5:
raise ValueError('md5 passed to write_fd does not match fd data')
resp = self.conn.read_response()
# If we're really unlucky, then the token has expired while we were uploading data.
if resp.status == 401:
self.conn.discard()
raise AccessTokenExpired()
elif resp.status != 200:
exc = self._parse_error_response(resp)
raise _map_request_error(exc, key) or exc
self._parse_json_response(resp)
def _send_request(self,
method,
path,
headers,
subres=None,
query_string=None,
body=None):
'''Add authentication and send request
Returns the response object.
'''
if not isinstance(headers, CaseInsensitiveDict):
headers = CaseInsensitiveDict(headers)
self._authorize_request(method, path, headers, subres)
# Construct full path
if not self.hostname.startswith(self.bucket_name):
path = '/%s%s' % (self.bucket_name, path)
path = urllib.parse.quote(path)
if query_string:
s = urllib.parse.urlencode(query_string, doseq=True)
if subres:
path += '?%s&%s' % (subres, s)
else:
path += '?%s' % s
elif subres:
path += '?%s' % subres
# We can probably remove the assertions at some point and
# call self.conn.read_response() directly
def read_response():
resp = self.conn.read_response()
assert resp.method == method
assert resp.path == path
return resp
use_expect_100c = not self.options.get('disable-expect100', False)
try:
log.debug('sending %s %s', method, path)
if body is None or isinstance(body,
(bytes, bytearray, memoryview)):
self.conn.send_request(method,
path,
body=body,
headers=headers)
else:
body_len = os.fstat(body.fileno()).st_size
self.conn.send_request(method,
path,
expect100=use_expect_100c,
headers=headers,
body=BodyFollowing(body_len))
if use_expect_100c:
resp = read_response()
if resp.status != 100: # Error
return resp
try:
copyfileobj(body, self.conn, BUFSIZE)
except ConnectionClosed:
# Server closed connection while we were writing body data -
# but we may still be able to read an error response
try:
resp = read_response()
except ConnectionClosed: # No server response available
pass
else:
if resp.status >= 400: # Got error response
return resp
log.warning(
'Server broke connection during upload, but signaled '
'%d %s', resp.status, resp.reason)
# Re-raise first ConnectionClosed exception
raise
return read_response()
except Exception as exc:
if is_temp_network_error(exc):
# We probably can't use the connection anymore
self.conn.disconnect()
raise
def test_write_toolittle2(conn):
conn.send_request('PUT', '/allgood', body=BodyFollowing(42))
conn.write(DUMMY_DATA[:24])
with pytest.raises(dugong.StateError):
conn.read_response()
def test_write_toomuch(conn):
conn.send_request('PUT', '/allgood', body=BodyFollowing(42))
with pytest.raises(dugong.ExcessBodyData):
conn.write(DUMMY_DATA[:43])
def test_abort_write(conn):
conn.send_request('PUT', '/allgood', body=BodyFollowing(42))
conn.write(b'fooo')
conn.disconnect()
assert_raises(dugong.ConnectionClosed, conn.write, b'baar')
def _do_request(self,
connection,
method,
path,
headers=None,
body=None,
download_body=True):
'''Send request, read and return response object'''
log.debug('started with %s %s', method, path)
if headers is None:
headers = CaseInsensitiveDict()
if self.authorization_token is None:
self._authorize_account()
if 'Authorization' not in headers:
headers['Authorization'] = self.authorization_token
if self.test_mode_expire_some_tokens:
headers[
'X-Bz-Test-Mode'] = 'expire_some_account_authorization_tokens'
if self.test_mode_force_cap_exceeded:
headers['X-Bz-Test-Mode'] = 'force_cap_exceeded'
log.debug('REQUEST: %s %s %s', connection.hostname, method, path)
if body is None or isinstance(body, (bytes, bytearray, memoryview)):
connection.send_request(method, path, headers=headers, body=body)
else:
body_length = os.fstat(body.fileno()).st_size
connection.send_request(method,
path,
headers=headers,
body=BodyFollowing(body_length))
copyfileobj(body, connection, BUFSIZE)
response = connection.read_response()
if download_body is True or response.status != 200: # Backblaze always returns a json with error information in body
response_body = connection.readall()
else:
response_body = None
content_length = response.headers.get('Content-Length', '0')
log.debug('RESPONSE: %s %s %s %s', response.method, response.status,
response.reason, content_length)
if (
response.status == 404 or # File not found
(response.status != 200 and method == 'HEAD')
): # HEAD responses do not have a body -> we have to raise a HTTPError with the code
raise HTTPError(response.status, response.reason, response.headers)
if response.status != 200:
json_error_response = json.loads(
response_body.decode('utf-8')) if response_body else None
code = json_error_response['code'] if json_error_response else None
message = json_error_response[
'message'] if json_error_response else response.reason
b2_error = B2Error(json_error_response['status'], code, message,
response.headers)
raise b2_error
return response, response_body
def _do_request(self,
method,
path,
conn,
headers=None,
body=None,
auth_token=None,
download_body=True,
body_size=None):
"""Send request, read and return response object
This method modifies the *headers* dictionary.
conn must by a HTTPConnection
When download_body is True, need to receive data before making new connection
"""
def _debug_body(b):
if isinstance(b, str):
return b
elif b is None:
return "None"
else:
return 'byte_body'
def _debug_hostname(c):
try:
return c.hostname
except:
return "None"
log.debug('started with %r, %r, %r, %r, %r', method,
_debug_hostname(conn), path, headers, _debug_body(body))
if headers is None:
headers = CaseInsensitiveDict()
if auth_token is None:
headers['Authorization'] = self.auth_token
else:
headers['Authorization'] = auth_token
if self.test_string:
headers['X-Bz-Test-Mode'] = self.test_string
try:
if isinstance(body, io.FileIO):
if body_size is None:
raise ValueError(
"Body size is necessary when uploading from file")
conn.send_request(method,
path,
headers=headers,
body=BodyFollowing(body_size))
while True:
buf = body.read(BUFSIZE)
if not buf:
break
conn.write(buf)
else:
conn.send_request(method, path, headers=headers, body=body)
resp = conn.read_response()
if download_body or resp.status != 200:
body = conn.readall()
else:
# caller need to download body itself before making new request
body = None
except Exception as exc:
if is_temp_network_error(exc):
# We probably can't use the connection anymore
conn.disconnect()
raise
if resp.status == 200 or resp.status == 206:
return resp, body
try:
# error code is in body
j = json.loads(str(body, encoding='UTF-8'))
except ValueError:
raise HTTPError(resp.status, resp.reason, resp.headers)
# Expired auth token
if resp.status == 401:
if j['code'] == 'expired_auth_token':
log.info(
'BackBlaze auth token seems to have expired, requesting new one.'
)
self.conn_api.disconnect()
self.conn_download.disconnect()
# Force constructing a new connection with a new token, otherwise
# the connection will be reestablished with the same token.
self.conn_api = None
self.conn_download = None
self._login()
raise AuthenticationExpired(j['message'])
else:
raise AuthorizationError(j['message'])
# File not found
if resp.status == 404:
raise NoSuchObject(path)
# Backend error
raise B2Error(j['status'], j['code'], j['message'], headers=headers)
