def test_http_proxy(http_server, monkeypatch, test_port):
test_host = 'www.foobarz.invalid'
test_path = '/someurl?barf'
get_path = None
def do_GET(self):
nonlocal get_path
get_path = self.path
self.send_response(200)
self.send_header("Content-Type", 'application/octet-stream')
self.send_header("Content-Length", '0')
self.end_headers()
monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET)
conn = HTTPConnection(test_host, test_port,
proxy=(http_server.host, http_server.port))
try:
conn.send_request('GET', test_path)
resp = conn.read_response()
assert resp.status == 200
conn.discard()
finally:
conn.disconnect()
if test_port is None:
exp_path = 'http://%s%s' % (test_host, test_path)
else:
exp_path = 'http://%s:%d%s' % (test_host, test_port, test_path)
assert get_path == exp_path
def test_http_proxy(http_server, monkeypatch, test_port):
test_host = 'www.foobarz.invalid'
test_path = '/someurl?barf'
get_path = None
def do_GET(self):
nonlocal get_path
get_path = self.path
self.send_response(200)
self.send_header("Content-Type", 'application/octet-stream')
self.send_header("Content-Length", '0')
self.end_headers()
monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET)
conn = HTTPConnection(test_host,
test_port,
proxy=(http_server.host, http_server.port))
try:
conn.send_request('GET', test_path)
resp = conn.read_response()
assert resp.status == 200
conn.discard()
finally:
conn.disconnect()
if test_port is None:
exp_path = 'http://%s%s' % (test_host, test_path)
else:
exp_path = 'http://%s:%d%s' % (test_host, test_port, test_path)
assert get_path == exp_path
def _get_access_token(self):
log.info('Requesting new access token')
headers = CaseInsensitiveDict()
headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=utf-8'
body = urllib.parse.urlencode({
'client_id': OAUTH_CLIENT_ID,
'client_secret': OAUTH_CLIENT_SECRET,
'refresh_token': self.password,
'grant_type': 'refresh_token' })
conn = HTTPConnection('accounts.google.com', 443, proxy=self.proxy,
ssl_context=self.ssl_context)
try:
conn.send_request('POST', '/o/oauth2/token', headers=headers,
body=body.encode('utf-8'))
resp = conn.read_response()
json_resp = self._parse_json_response(resp, conn)
if resp.status > 299 or resp.status < 200:
assert 'error' in json_resp
if 'error' in json_resp:
raise AuthenticationError(json_resp['error'])
else:
self.access_token[self.password] = json_resp['access_token']
finally:
conn.disconnect()
def test_invalid_ssl():
check_http_connection()
# Don't load certificates
context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
context.options |= ssl.OP_NO_SSLv2
context.verify_mode = ssl.CERT_REQUIRED
conn = HTTPConnection(SSL_TEST_HOST, ssl_context=context)
with pytest.raises(ssl.SSLError):
conn.send_request('GET', '/')
conn.disconnect()
def _get_access_token(self):
log.info('Requesting new access token')
headers = CaseInsensitiveDict()
headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=utf-8'
body = urlencode({'client_id': OAUTH_CLIENT_ID,
'client_secret': OAUTH_CLIENT_SECRET,
'refresh_token': self.password,
'grant_type': 'refresh_token' })
conn = HTTPConnection('accounts.google.com', 443, proxy=self.proxy,
ssl_context=self.ssl_context)
try:
conn.send_request('POST', '/o/oauth2/token', headers=headers,
body=body.encode('utf-8'))
resp = conn.read_response()
if resp.status > 299 or resp.status < 200:
raise HTTPError(resp.status, resp.reason, resp.headers)
content_type = resp.headers.get('Content-Type', None)
if content_type:
hit = re.match(r'application/json(?:; charset="(.+)")?$',
resp.headers['Content-Type'], re.IGNORECASE)
else:
hit = None
if not hit:
log.error('Unexpected server reply when refreshing access token:\n%s',
self._dump_response(resp))
raise RuntimeError('Unable to parse server response')
charset = hit.group(1) or 'utf-8'
body = conn.readall().decode(charset)
resp_json = json.loads(body)
if not isinstance(resp_json, dict):
log.error('Invalid json server response. Expected dict, got:\n%s', body)
raise RuntimeError('Unable to parse server response')
if 'error' in resp_json:
raise AuthenticationError(resp_json['error'])
if 'access_token' not in resp_json:
log.error('Unable to find access token in server response:\n%s', body)
raise RuntimeError('Unable to parse server response')
self.access_token[self.password] = resp_json['access_token']
finally:
conn.disconnect()
def test_invalid_ssl():
check_http_connection()
# Don't load certificates
context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
context.options |= ssl.OP_NO_SSLv2
context.verify_mode = ssl.CERT_REQUIRED
conn = HTTPConnection(SSL_TEST_HOST, ssl_context=context)
with pytest.raises(ssl.SSLError):
conn.send_request('GET', '/')
conn.disconnect()
def test_connect_proxy(http_server, monkeypatch, test_port):
test_host = 'www.foobarz.invalid'
test_path = '/someurl?barf'
connect_path = None
def do_CONNECT(self):
# Pretend we're the remote server too
nonlocal connect_path
connect_path = self.path
self.send_response(200)
self.end_headers()
self.close_connection = 0
monkeypatch.setattr(MockRequestHandler,
'do_CONNECT',
do_CONNECT,
raising=False)
get_path = None
def do_GET(self):
nonlocal get_path
get_path = self.path
self.send_response(200)
self.send_header("Content-Type", 'application/octet-stream')
self.send_header("Content-Length", '0')
self.end_headers()
monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET)
# We don't *actually* want to establish SSL, that'd be
# to complex for our mock server
monkeypatch.setattr('ssl.match_hostname', lambda x, y: True)
conn = HTTPConnection(test_host,
test_port,
proxy=(http_server.host, http_server.port),
ssl_context=FakeSSLContext())
try:
conn.send_request('GET', test_path)
resp = conn.read_response()
assert resp.status == 200
conn.discard()
finally:
conn.disconnect()
if test_port is None:
test_port = 443
exp_path = '%s:%d' % (test_host, test_port)
assert connect_path == exp_path
assert get_path == test_path
def test_connect_ssl():
check_http_connection()
ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
ssl_context.options |= ssl.OP_NO_SSLv2
ssl_context.verify_mode = ssl.CERT_REQUIRED
ssl_context.set_default_verify_paths()
conn = HTTPConnection(SSL_TEST_HOST, ssl_context=ssl_context)
conn.send_request('GET', '/')
resp = conn.read_response()
assert resp.status in (200, 301, 302)
assert resp.path == '/'
conn.discard()
conn.disconnect()
def test_connect_ssl():
check_http_connection()
ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
ssl_context.options |= ssl.OP_NO_SSLv2
ssl_context.verify_mode = ssl.CERT_REQUIRED
ssl_context.set_default_verify_paths()
conn = HTTPConnection(SSL_TEST_HOST, ssl_context=ssl_context)
conn.send_request('GET', '/')
resp = conn.read_response()
assert resp.status in (200, 301, 302)
assert resp.path == '/'
conn.discard()
conn.disconnect()
def test_connect_proxy(http_server, monkeypatch, test_port):
test_host = 'www.foobarz.invalid'
test_path = '/someurl?barf'
connect_path = None
def do_CONNECT(self):
# Pretend we're the remote server too
nonlocal connect_path
connect_path = self.path
self.send_response(200)
self.end_headers()
self.close_connection = 0
monkeypatch.setattr(MockRequestHandler, 'do_CONNECT',
do_CONNECT, raising=False)
get_path = None
def do_GET(self):
nonlocal get_path
get_path = self.path
self.send_response(200)
self.send_header("Content-Type", 'application/octet-stream')
self.send_header("Content-Length", '0')
self.end_headers()
monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET)
# We don't *actually* want to establish SSL, that'd be
# to complex for our mock server
monkeypatch.setattr('ssl.match_hostname', lambda x,y: True)
conn = HTTPConnection(test_host, test_port,
proxy=(http_server.host, http_server.port),
ssl_context=FakeSSLContext())
try:
conn.send_request('GET', test_path)
resp = conn.read_response()
assert resp.status == 200
conn.discard()
finally:
conn.disconnect()
if test_port is None:
test_port = 443
exp_path = '%s:%d' % (test_host, test_port)
assert connect_path == exp_path
assert get_path == test_path
class Backend(AbstractBackend, metaclass=ABCDocstMeta):
needs_login = True
known_options = {'test-string', 'ssl-ca-path'}
authorize_url = "/b2api/v1/b2_authorize_account"
authorize_hostname = "api.backblaze.com"
hdr_prefix = 'X-Bz-Info-'
# no chunksize limit on backend, so we try to limit the number of meta headers
hdr_chunksize = 2000
_add_meta_headers_s3 = s3c.Backend._add_meta_headers
_extractmeta_s3 = s3c.Backend._extractmeta
def __init__(self, storage_url, login, password, options):
super().__init__()
self.options = options
self.bucket_name = None
self.prefix = None
self.auth_token = None
self.api_host = None
self.download_host = None
self.conn_api = None
self.conn_download = None
self.account_id = login
self.account_key = password
self.bucket_id = None
self.bucket_name = None
self._parse_storage_url(storage_url)
# Add test header povided to all requests.
if options.get('test-string'):
# sanitize entry string
self.test_string = ''.join([
x for x in options.get('test-string')
if x in 'abcdefghijklmnopqrstuvwxyz_'
])
else:
self.test_string = None
self.ssl_context = get_ssl_context(options.get('ssl-ca-path', None))
self._login()
self._connect_bucket(self.bucket_name)
def _do_request(self,
method,
path,
conn,
headers=None,
body=None,
auth_token=None,
download_body=True,
body_size=None):
"""Send request, read and return response object
This method modifies the *headers* dictionary.
conn must by a HTTPConnection
When download_body is True, need to receive data before making new connection
"""
def _debug_body(b):
if isinstance(b, str):
return b
elif b is None:
return "None"
else:
return 'byte_body'
def _debug_hostname(c):
try:
return c.hostname
except:
return "None"
log.debug('started with %r, %r, %r, %r, %r', method,
_debug_hostname(conn), path, headers, _debug_body(body))
if headers is None:
headers = CaseInsensitiveDict()
if auth_token is None:
headers['Authorization'] = self.auth_token
else:
headers['Authorization'] = auth_token
if self.test_string:
headers['X-Bz-Test-Mode'] = self.test_string
try:
if isinstance(body, io.FileIO):
if body_size is None:
raise ValueError(
"Body size is necessary when uploading from file")
conn.send_request(method,
path,
headers=headers,
body=BodyFollowing(body_size))
while True:
buf = body.read(BUFSIZE)
if not buf:
break
conn.write(buf)
else:
conn.send_request(method, path, headers=headers, body=body)
resp = conn.read_response()
if download_body or resp.status != 200:
body = conn.readall()
else:
# caller need to download body itself before making new request
body = None
except Exception as exc:
if is_temp_network_error(exc):
# We probably can't use the connection anymore
conn.disconnect()
raise
if resp.status == 200 or resp.status == 206:
return resp, body
try:
# error code is in body
j = json.loads(str(body, encoding='UTF-8'))
except ValueError:
raise HTTPError(resp.status, resp.reason, resp.headers)
# Expired auth token
if resp.status == 401:
if j['code'] == 'expired_auth_token':
log.info(
'BackBlaze auth token seems to have expired, requesting new one.'
)
self.conn_api.disconnect()
self.conn_download.disconnect()
# Force constructing a new connection with a new token, otherwise
# the connection will be reestablished with the same token.
self.conn_api = None
self.conn_download = None
self._login()
raise AuthenticationExpired(j['message'])
else:
raise AuthorizationError(j['message'])
# File not found
if resp.status == 404:
raise NoSuchObject(path)
# Backend error
raise B2Error(j['status'], j['code'], j['message'], headers=headers)
def _login(self):
"""
Login with backend and make a new connection to API and download server
"""
id_and_key = self.account_id + ':' + self.account_key
basic_auth_string = 'Basic ' + str(
base64.b64encode(bytes(id_and_key, 'UTF-8')), encoding='UTF-8')
with HTTPConnection(self.authorize_hostname,
443,
ssl_context=self.ssl_context) as conn:
resp, body = self._do_request('GET',
self.authorize_url,
conn,
auth_token=basic_auth_string)
j = json.loads(str(body, encoding='UTF-8'))
api_url = urllib.parse.urlparse(j['apiUrl'])
download_url = urllib.parse.urlparse(j['downloadUrl'])
self.api_host = api_url.hostname
self.auth_token = j['authorizationToken']
self.download_host = download_url.hostname
self.conn_api = HTTPConnection(self.api_host,
443,
ssl_context=self.ssl_context)
self.conn_download = HTTPConnection(self.download_host,
443,
ssl_context=self.ssl_context)
def _connect_bucket(self, bucket_name):
"""
Get id of bucket_name
"""
log.debug('started with %s' % (bucket_name))
resp, body = self._do_request(
'GET', '/b2api/v1/b2_list_buckets?accountId=%s' % self.account_id,
self.conn_api)
bucket_id = None
j = json.loads(str(body, encoding='UTF-8'))
for b in j['buckets']:
if b['bucketName'] == bucket_name:
bucket_id = b['bucketId']
if bucket_id is None:
raise DanglingStorageURLError(bucket_name)
self.bucket_id = bucket_id
self.bucket_name = bucket_name
def _add_meta_headers(self, headers, metadata):
self._add_meta_headers_s3(headers,
metadata,
chunksize=self.hdr_chunksize)
# URL encode headers
for i in count():
# Headers is an email.message object, so indexing it
# would also give None instead of KeyError
key = '%smeta-%03d' % (self.hdr_prefix, i)
part = headers.get(key, None)
if part is None:
break
headers[key] = urllib.parse.quote(part.encode('utf-8'))
# Check we dont reach the metadata backend lmits
if i > 10:
raise RuntimeError("Too metadata for the backend")
def _extractmeta(self, resp, filename):
# URL decode headers
for i in count():
# Headers is an email.message object, so indexing it
# would also give None instead of KeyError
key = '%smeta-%03d' % (self.hdr_prefix, i)
part = resp.headers.get(key, None)
if part is None:
break
resp.headers.replace_header(key,
urllib.parse.unquote_plus(str(part)))
return self._extractmeta_s3(resp, filename)
def _get_upload_url(self):
"""Get a single use URL to upload a file"""
log.debug('started')
body = bytes(json.dumps({'bucketId': self.bucket_id}),
encoding='UTF-8')
resp, body = self._do_request('POST',
'/b2api/v1/b2_get_upload_url',
self.conn_api,
body=body)
j = json.loads(str(body, encoding='UTF-8'))
return j['authorizationToken'], j['uploadUrl']
def _parse_storage_url(self, storage_url):
"""Init instance variables from storage url"""
hit = re.match(r'^b2?://([^/]+)(?:/(.*))?$', storage_url)
if not hit:
raise QuietError('Invalid storage URL', exitcode=2)
bucket_name = hit.group(1)
if not re.match('^(?!b2-)[a-z0-9A-Z\-]{6,50}$', bucket_name):
raise QuietError('Invalid bucket name.', exitcode=2)
prefix = hit.group(2) or ''
#remove trailing slash if exist
prefix = prefix[:-1] if prefix[-1] == '/' else prefix
self.bucket_name = bucket_name
self.prefix = prefix
def _delete_file_id(self, filelist):
for file in filelist:
body = bytes(json.dumps({
'fileName': file['fileName'],
'fileId': file['fileId']
}),
encoding='UTF-8')
log.debug('started with /file/%s/%s/%s' %
(self.bucket_name, self.prefix, file['fileName']))
try:
self._do_request('POST',
'/b2api/v1/b2_delete_file_version',
self.conn_api,
body=body)
except B2Error as err:
# Server may return file_not_present
# just let it close connection and retry
if err.code == 'file_not_present':
pass
else:
raise err
@retry
def _list_file_version(self, key, max_filecount=1000):
if max_filecount > 1000:
raise ValueError('max_filecount maximum is 1000')
request_dict = dict(bucketId=self.bucket_id,
maxFileCount=max_filecount)
request_dict['startFileName'] = self.prefix + '/' + self._encode_key(
key)
# supposing there is less than 1000 old file version
body = bytes(json.dumps(request_dict), encoding='UTF-8')
resp, body = self._do_request('POST',
'/b2api/v1/b2_list_file_versions',
self.conn_api,
body=body)
j = json.loads(str(body, encoding='UTF-8'))
r = []
# We suppose there is less than 1000 file version
for f in j['files']:
if self._decode_key(f['fileName']) == self.prefix + '/' + key:
r.append({'fileName': f['fileName'], 'fileId': f['fileId']})
return r
@retry
def _list_file_name(self, start_filename=None, max_filecount=1000):
if max_filecount > 1000:
raise ValueError('max_filecount maximum is 1000')
request_dict = {
'bucketId': self.bucket_id,
'maxFileCount': max_filecount
}
if start_filename is not None:
request_dict['startFileName'] = start_filename
body = bytes(json.dumps(request_dict), encoding='UTF-8')
resp, body = self._do_request('POST',
'/b2api/v1/b2_list_file_names',
self.conn_api,
body=body)
j = json.loads(str(body, encoding='UTF-8'))
filelist = [f['fileName'] for f in j['files']]
return j['nextFileName'], filelist
def _encode_key(self, filename):
# URLencode filename
filename = urllib.parse.quote(filename.encode('utf-8'), safe='/\\')
# DIRTY HACK :
# Backend does not support backslashes, we change them to pass test
filename = filename.replace("\\", "__")
return filename
def _decode_key(self, filename):
# DIRTY HACK :
# Backend does not support backslashes, we change them to pass test
filename = filename.replace("__", "\\")
# URLencode filename
filename = urllib.parse.unquote(filename)
return filename
def has_native_rename(self):
"""True if the backend has a native, atomic rename operation"""
return False
@copy_ancestor_docstring
def is_temp_failure(self, exc):
if isinstance(exc, AuthenticationExpired):
return True
if isinstance(exc, ssl.SSLError):
return True
if is_temp_network_error(exc):
return True
if isinstance(exc, HTTPError):
return True
if isinstance(exc, B2Error):
if (exc.status == 400 or \
exc.status == 408 or \
(exc.status >= 500 and exc.status <= 599)):
return True
return False
@retry
@copy_ancestor_docstring
def lookup(self, key):
log.debug('started with %s', key)
key = self._encode_key(key)
headers = CaseInsensitiveDict()
headers['Range'] = "bytes=0-1" # Only get first byte
resp, data = self._do_request('GET',
'/file/%s/%s/%s' %
(self.bucket_name, self.prefix, key),
self.conn_download,
headers=headers)
meta = self._extractmeta(resp, key)
return meta
@retry
def get_size(self, key):
"""Return size of object stored under *key*"""
log.debug('started with %s', key)
key = self._encode_key(key)
key_with_prefix = "%s/%s" % (self.prefix, key)
request_dict = {
'bucketId': self.bucket_id,
'startFileName': key_with_prefix,
'maxFileCount': 1
}
body = bytes(json.dumps(request_dict), encoding='UTF-8')
resp, body = self._do_request('POST',
'/b2api/v1/b2_list_file_names',
self.conn_api,
body=body)
j = json.loads(str(body, encoding='UTF-8'))
if j['files'][0]['fileName'] == key_with_prefix:
return j['files'][0]['contentLength']
raise NoSuchObject(key_with_prefix)
@retry
@copy_ancestor_docstring
def open_read(self, key):
log.debug('started with %s', key)
key = self._encode_key(key)
resp, data = self._do_request('GET',
'/file/%s/%s/%s' %
(self.bucket_name, self.prefix, key),
self.conn_download,
download_body=False)
meta = self._extractmeta(resp, key)
return ObjectR(key, self.conn_download, resp, metadata=meta)
@copy_ancestor_docstring
def open_write(self, key, metadata=None, is_compressed=False):
log.debug('started with %s', key)
key = self._encode_key(key)
return ObjectW(key, self, metadata)
@retry
@copy_ancestor_docstring
def clear(self):
log.debug('started')
for file in self.list():
self.delete(file, force=True)
@retry
@copy_ancestor_docstring
def delete(self, key, force=False):
log.debug('started with %s', key)
todel_id = self._list_file_version(key)
if not todel_id and not force:
raise NoSuchObject(key)
self._delete_file_id(todel_id)
@copy_ancestor_docstring
def list(self, prefix=''):
log.debug('started with %s', prefix)
prefix = self._encode_key(prefix)
next_filename = self.prefix + '/' + prefix
keys_remaining = True
while keys_remaining and next_filename is not None:
next_filename, filelist = self._list_file_name(next_filename)
while filelist:
file = filelist.pop(0)
if file.startswith(self.prefix + '/' + prefix):
# remove prefix before return
r = file[len(self.prefix + '/'):]
yield self._decode_key(r)
else:
keys_remaining = False
break
@prepend_ancestor_docstring
def copy(self, src, dest, metadata=None):
"""No atomic copy operation on backend. Hope this does not append to often"""
log.debug('started with %s, %s', src, dest)
data, src_meta = self.fetch(src)
# Delete dest file if already exist
self.delete(dest, force=True)
if metadata is None:
dst_meta = src_meta
else:
dst_meta = metadata
self.store(dest, data, dst_meta)
@copy_ancestor_docstring
def update_meta(self, key, metadata):
log.debug('started with %s', key)
self.copy(key, key, metadata)
def close(self):
log.debug('started')
self.conn_api.disconnect()
self.conn_download.disconnect()