def _get_conn(self):
'''Return connection to server'''
conn = HTTPConnection(self.hostname, self.port, proxy=self.proxy,
ssl_context=self.ssl_context)
conn.timeout = int(self.options.get('tcp-timeout', 20))
return conn
def _login(self):
"""
Login with backend and make a new connection to API and download server
"""
id_and_key = self.account_id + ':' + self.account_key
basic_auth_string = 'Basic ' + str(
base64.b64encode(bytes(id_and_key, 'UTF-8')), encoding='UTF-8')
with HTTPConnection(self.authorize_hostname,
443,
ssl_context=self.ssl_context) as conn:
resp, body = self._do_request('GET',
self.authorize_url,
conn,
auth_token=basic_auth_string)
j = json.loads(str(body, encoding='UTF-8'))
api_url = urllib.parse.urlparse(j['apiUrl'])
download_url = urllib.parse.urlparse(j['downloadUrl'])
self.api_host = api_url.hostname
self.auth_token = j['authorizationToken']
self.download_host = download_url.hostname
self.conn_api = HTTPConnection(self.api_host,
443,
ssl_context=self.ssl_context)
self.conn_download = HTTPConnection(self.download_host,
443,
ssl_context=self.ssl_context)
def _get_conn(self):
'''Return connection to server'''
conn = HTTPConnection(self.hostname, self.port, proxy=self.proxy,
ssl_context=self.ssl_context)
conn.timeout = int(self.options.get('tcp-timeout', 20))
return conn
def _get_conn(self):
'''Obtain connection to server and authentication token'''
log.debug('started')
if 'no-ssl' in self.options:
ssl_context = None
else:
ssl_context = self.ssl_context
headers = CaseInsensitiveDict()
headers['X-Auth-User'] = self.login
headers['X-Auth-Key'] = self.password
with HTTPConnection(self.hostname,
self.port,
proxy=self.proxy,
ssl_context=ssl_context) as conn:
conn.timeout = int(self.options.get('tcp-timeout', 20))
for auth_path in ('/v1.0', '/auth/v1.0'):
log.debug('GET %s', auth_path)
conn.send_request('GET', auth_path, headers=headers)
resp = conn.read_response()
if resp.status in (404, 412):
log.debug('auth to %s failed, trying next path', auth_path)
conn.discard()
continue
elif resp.status == 401:
raise AuthorizationError(resp.reason)
elif resp.status > 299 or resp.status < 200:
raise HTTPError(resp.status, resp.reason, resp.headers)
# Pylint can't infer SplitResult Types
#pylint: disable=E1103
self.auth_token = resp.headers['X-Auth-Token']
o = urlsplit(resp.headers['X-Storage-Url'])
self.auth_prefix = urllib.parse.unquote(o.path)
if o.scheme == 'https':
ssl_context = self.ssl_context
elif o.scheme == 'http':
ssl_context = None
else:
# fall through to scheme used for authentication
pass
conn = HTTPConnection(o.hostname,
o.port,
proxy=self.proxy,
ssl_context=ssl_context)
conn.timeout = int(self.options.get('tcp-timeout', 20))
return conn
raise RuntimeError('No valid authentication path found')
def test_http_proxy(http_server, monkeypatch, test_port):
test_host = 'www.foobarz.invalid'
test_path = '/someurl?barf'
get_path = None
def do_GET(self):
nonlocal get_path
get_path = self.path
self.send_response(200)
self.send_header("Content-Type", 'application/octet-stream')
self.send_header("Content-Length", '0')
self.end_headers()
monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET)
conn = HTTPConnection(test_host,
test_port,
proxy=(http_server.host, http_server.port))
try:
conn.send_request('GET', test_path)
resp = conn.read_response()
assert resp.status == 200
conn.discard()
finally:
conn.disconnect()
if test_port is None:
exp_path = 'http://%s%s' % (test_host, test_path)
else:
exp_path = 'http://%s:%d%s' % (test_host, test_port, test_path)
assert get_path == exp_path
def _get_conn(self):
'''Obtain connection to server and authentication token'''
log.debug('started')
if 'no-ssl' in self.options:
ssl_context = None
else:
ssl_context = self.ssl_context
headers = CaseInsensitiveDict()
headers['X-Auth-User'] = self.login
headers['X-Auth-Key'] = self.password
with HTTPConnection(self.hostname, self.port, proxy=self.proxy,
ssl_context=ssl_context) as conn:
conn.timeout = int(self.options.get('tcp-timeout', 10))
for auth_path in ('/v1.0', '/auth/v1.0'):
log.debug('GET %s', auth_path)
conn.send_request('GET', auth_path, headers=headers)
resp = conn.read_response()
if resp.status in (404, 412):
log.debug('auth to %s failed, trying next path', auth_path)
conn.discard()
continue
elif resp.status == 401:
raise AuthorizationError(resp.reason)
elif resp.status > 299 or resp.status < 200:
raise HTTPError(resp.status, resp.reason, resp.headers)
# Pylint can't infer SplitResult Types
#pylint: disable=E1103
self.auth_token = resp.headers['X-Auth-Token']
o = urlsplit(resp.headers['X-Storage-Url'])
self.auth_prefix = urllib.parse.unquote(o.path)
if o.scheme == 'https':
ssl_context = self.ssl_context
elif o.scheme == 'http':
ssl_context = None
else:
# fall through to scheme used for authentication
pass
conn = HTTPConnection(o.hostname, o.port, proxy=self.proxy,
ssl_context=ssl_context)
conn.timeout = int(self.options.get('tcp-timeout', 10))
return conn
raise RuntimeError('No valid authentication path found')
def test_invalid_ssl():
check_http_connection()
# Don't load certificates
context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
context.options |= ssl.OP_NO_SSLv2
context.verify_mode = ssl.CERT_REQUIRED
conn = HTTPConnection(SSL_TEST_HOST, ssl_context=context)
with pytest.raises(ssl.SSLError):
conn.send_request('GET', '/')
conn.disconnect()
def test_http_proxy(http_server, monkeypatch, test_port):
test_host = 'www.foobarz.invalid'
test_path = '/someurl?barf'
get_path = None
def do_GET(self):
nonlocal get_path
get_path = self.path
self.send_response(200)
self.send_header("Content-Type", 'application/octet-stream')
self.send_header("Content-Length", '0')
self.end_headers()
monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET)
conn = HTTPConnection(test_host, test_port,
proxy=(http_server.host, http_server.port))
try:
conn.send_request('GET', test_path)
resp = conn.read_response()
assert resp.status == 200
conn.discard()
finally:
conn.disconnect()
if test_port is None:
exp_path = 'http://%s%s' % (test_host, test_path)
else:
exp_path = 'http://%s:%d%s' % (test_host, test_port, test_path)
assert get_path == exp_path
def close(self):
"""Close object and upload data"""
log.debug('started with %s', self.key)
if self.closed:
# still call fh.close, may have generated an error before
self.fh.close()
return
self.fh.seek(0)
upload_auth_token, upload_url = self.backend._get_upload_url()
upload_url = urllib.parse.urlparse(upload_url)
with HTTPConnection(upload_url.hostname, 443, ssl_context=self.backend.ssl_context) as conn_up:
headers = CaseInsensitiveDict()
headers['X-Bz-File-Name'] = self.backend.prefix + self.key
headers['Content-Type'] = 'application/octet-stream'
headers['Content-Length'] = self.obj_size
headers['X-Bz-Content-Sha1'] = self.sha1.hexdigest()
if self.meta is None:
self.meta = dict()
self.backend._add_meta_headers(headers, self.meta)
self.backend._do_request('POST', upload_url.path + '?' + upload_url.query,
conn_up,
headers=headers,
body=self.fh,
auth_token=upload_auth_token,
body_size=self.obj_size)
self.fh.close()
self.closed = True
def test_connect_proxy(http_server, monkeypatch, test_port):
test_host = 'www.foobarz.invalid'
test_path = '/someurl?barf'
connect_path = None
def do_CONNECT(self):
# Pretend we're the remote server too
nonlocal connect_path
connect_path = self.path
self.send_response(200)
self.end_headers()
self.close_connection = 0
monkeypatch.setattr(MockRequestHandler,
'do_CONNECT',
do_CONNECT,
raising=False)
get_path = None
def do_GET(self):
nonlocal get_path
get_path = self.path
self.send_response(200)
self.send_header("Content-Type", 'application/octet-stream')
self.send_header("Content-Length", '0')
self.end_headers()
monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET)
# We don't *actually* want to establish SSL, that'd be
# to complex for our mock server
monkeypatch.setattr('ssl.match_hostname', lambda x, y: True)
conn = HTTPConnection(test_host,
test_port,
proxy=(http_server.host, http_server.port),
ssl_context=FakeSSLContext())
try:
conn.send_request('GET', test_path)
resp = conn.read_response()
assert resp.status == 200
conn.discard()
finally:
conn.disconnect()
if test_port is None:
test_port = 443
exp_path = '%s:%d' % (test_host, test_port)
assert connect_path == exp_path
assert get_path == test_path
def _get_access_token(self):
log.info('Requesting new access token')
headers = CaseInsensitiveDict()
headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=utf-8'
body = urllib.parse.urlencode({
'client_id': OAUTH_CLIENT_ID,
'client_secret': OAUTH_CLIENT_SECRET,
'refresh_token': self.password,
'grant_type': 'refresh_token' })
conn = HTTPConnection('accounts.google.com', 443, proxy=self.proxy,
ssl_context=self.ssl_context)
try:
conn.send_request('POST', '/o/oauth2/token', headers=headers,
body=body.encode('utf-8'))
resp = conn.read_response()
json_resp = self._parse_json_response(resp, conn)
if resp.status > 299 or resp.status < 200:
assert 'error' in json_resp
if 'error' in json_resp:
raise AuthenticationError(json_resp['error'])
else:
self.access_token[self.password] = json_resp['access_token']
finally:
conn.disconnect()
def test_connect_ssl():
check_http_connection()
ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
ssl_context.options |= ssl.OP_NO_SSLv2
ssl_context.verify_mode = ssl.CERT_REQUIRED
ssl_context.set_default_verify_paths()
conn = HTTPConnection(SSL_TEST_HOST, ssl_context=ssl_context)
conn.send_request('GET', '/')
resp = conn.read_response()
assert resp.status in (200, 301, 302)
assert resp.path == '/'
conn.discard()
conn.disconnect()
def _get_download_connection(self):
if self.download_url is None:
self._authorize_account()
if self.download_connection is None:
self.download_connection = HTTPConnection(
self.download_url.hostname, 443, ssl_context=self.ssl_context)
self.download_connection.timeout = self.tcp_timeout
return self.download_connection
def _request_upload_url_info(self):
request_data = {'bucketId': self._get_bucket_id()}
response = self._do_api_call('b2_get_upload_url', request_data)
new_upload_url = urlparse(response['uploadUrl'])
new_authorization_token = response['authorizationToken']
upload_connection = HTTPConnection(new_upload_url.hostname,
443,
ssl_context=self.ssl_context)
upload_connection.timeout = self.tcp_timeout
return {
'hostname': new_upload_url.hostname,
'connection': upload_connection,
'path': new_upload_url.path,
'authorizationToken': new_authorization_token,
'isUploading': False
}
def _get_access_token(self):
log.info('Requesting new access token')
headers = CaseInsensitiveDict()
headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=utf-8'
body = urlencode({'client_id': OAUTH_CLIENT_ID,
'client_secret': OAUTH_CLIENT_SECRET,
'refresh_token': self.password,
'grant_type': 'refresh_token' })
conn = HTTPConnection('accounts.google.com', 443, proxy=self.proxy,
ssl_context=self.ssl_context)
try:
conn.send_request('POST', '/o/oauth2/token', headers=headers,
body=body.encode('utf-8'))
resp = conn.read_response()
if resp.status > 299 or resp.status < 200:
raise HTTPError(resp.status, resp.reason, resp.headers)
content_type = resp.headers.get('Content-Type', None)
if content_type:
hit = re.match(r'application/json(?:; charset="(.+)")?$',
resp.headers['Content-Type'], re.IGNORECASE)
else:
hit = None
if not hit:
log.error('Unexpected server reply when refreshing access token:\n%s',
self._dump_response(resp))
raise RuntimeError('Unable to parse server response')
charset = hit.group(1) or 'utf-8'
body = conn.readall().decode(charset)
resp_json = json.loads(body)
if not isinstance(resp_json, dict):
log.error('Invalid json server response. Expected dict, got:\n%s', body)
raise RuntimeError('Unable to parse server response')
if 'error' in resp_json:
raise AuthenticationError(resp_json['error'])
if 'access_token' not in resp_json:
log.error('Unable to find access token in server response:\n%s', body)
raise RuntimeError('Unable to parse server response')
self.access_token[self.password] = resp_json['access_token']
finally:
conn.disconnect()
def test_connect_ssl():
check_http_connection()
ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
ssl_context.options |= ssl.OP_NO_SSLv2
ssl_context.verify_mode = ssl.CERT_REQUIRED
ssl_context.set_default_verify_paths()
conn = HTTPConnection(SSL_TEST_HOST, ssl_context=ssl_context)
conn.send_request('GET', '/')
resp = conn.read_response()
assert resp.status in (200, 301, 302)
assert resp.path == '/'
conn.discard()
conn.disconnect()
def main():
if len(sys.argv) != 2:
raise SystemExit('Usage: %s <url>' % sys.argv[0])
url = sys.argv[1]
url_els = urlsplit(url)
if url_els.scheme == 'https':
ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
ssl_context.options |= ssl.OP_NO_SSLv2
ssl_context.verify_mode = ssl.CERT_REQUIRED
ssl_context.set_default_verify_paths()
else:
ssl_context = None
with HTTPConnection(url_els.hostname,
port=url_els.port,
ssl_context=ssl_context) as conn:
path = urlunsplit(('', '') + url_els[2:4] + ('', )) or '/'
conn.send_request('GET', path)
resp = conn.read_response()
if resp.status != 200:
raise SystemExit('%d %s' % (resp.status, resp.reason))
# Determine if we're reading text or binary data, and (in case of text),
# what character set is being used.
if 'Content-Type' not in resp.headers:
type_ = 'application/octet-stream'
else:
type_ = resp.headers['Content-Type']
hit = re.match(r'text/x?html(?:; charset=(.+))?$', type_)
if not hit:
raise SystemExit('Server did not send html but %s' % type_)
if hit.group(1):
charset = hit.group(1)
else:
charset = 'latin1'
html_stream = TextIOWrapper(conn, encoding=charset)
parser = LinkExtractor()
while True:
buf = html_stream.read(16 * 1024)
if not buf:
break
parser.feed(buf)
for link in parser.links:
print(urljoin(url, link))
def conn(request, http_server):
if http_server.use_ssl:
ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
ssl_context.options |= ssl.OP_NO_SSLv2
ssl_context.verify_mode = ssl.CERT_REQUIRED
ssl_context.load_verify_locations(
cafile=os.path.join(TEST_DIR, 'ca.crt'))
else:
ssl_context = None
conn = HTTPConnection(http_server.host,
port=http_server.port,
ssl_context=ssl_context)
request.addfinalizer(conn.disconnect)
return conn
def test_connect_proxy(http_server, monkeypatch, test_port):
test_host = 'www.foobarz.invalid'
test_path = '/someurl?barf'
connect_path = None
def do_CONNECT(self):
# Pretend we're the remote server too
nonlocal connect_path
connect_path = self.path
self.send_response(200)
self.end_headers()
self.close_connection = 0
monkeypatch.setattr(MockRequestHandler, 'do_CONNECT',
do_CONNECT, raising=False)
get_path = None
def do_GET(self):
nonlocal get_path
get_path = self.path
self.send_response(200)
self.send_header("Content-Type", 'application/octet-stream')
self.send_header("Content-Length", '0')
self.end_headers()
monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET)
# We don't *actually* want to establish SSL, that'd be
# to complex for our mock server
monkeypatch.setattr('ssl.match_hostname', lambda x,y: True)
conn = HTTPConnection(test_host, test_port,
proxy=(http_server.host, http_server.port),
ssl_context=FakeSSLContext())
try:
conn.send_request('GET', test_path)
resp = conn.read_response()
assert resp.status == 200
conn.discard()
finally:
conn.disconnect()
if test_port is None:
test_port = 443
exp_path = '%s:%d' % (test_host, test_port)
assert connect_path == exp_path
assert get_path == test_path
def test_invalid_ssl():
check_http_connection()
# Don't load certificates
context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
context.options |= ssl.OP_NO_SSLv2
context.verify_mode = ssl.CERT_REQUIRED
conn = HTTPConnection(SSL_TEST_HOST, ssl_context=context)
with pytest.raises(ssl.SSLError):
conn.send_request('GET', '/')
conn.disconnect()
def _authorize_account(self):
'''Authorize API calls'''
authorize_host = 'api.backblazeb2.com'
authorize_url = api_url_prefix + 'b2_authorize_account'
id_and_key = self.b2_application_key_id + ':' + self.b2_application_key
basic_auth_string = 'Basic ' + str(
base64.b64encode(bytes(id_and_key, 'UTF-8')), encoding='UTF-8')
with HTTPConnection(authorize_host, 443,
ssl_context=self.ssl_context) as connection:
headers = CaseInsensitiveDict()
headers['Authorization'] = basic_auth_string
connection.send_request('GET',
authorize_url,
headers=headers,
body=None)
response = connection.read_response()
response_body = connection.readall()
if response.status != 200:
raise RuntimeError('Authorization failed.')
j = json.loads(response_body.decode('utf-8'))
self.account_id = j['accountId']
allowed_info = j.get('allowed')
if allowed_info.get('bucketId'):
self.bucket_id = allowed_info.get('bucketId')
if allowed_info.get('bucketName') != self.bucket_name:
raise RuntimeError(
'Provided API key can not access desired bucket.')
if not self._check_key_capabilities(allowed_info):
raise RuntimeError(
'Provided API key does not have the required capabilities.'
)
self.api_url = urlparse(j['apiUrl'])
self.download_url = urlparse(j['downloadUrl'])
self.authorization_token = j['authorizationToken']
def __call__(self, url: str, method: str = 'GET', body=None,
headers=None, timeout=None):
# https://github.com/googleapis/google-auth-library-python/issues/318
if not isinstance(body, bytes):
body = str(body).encode('ascii')
if timeout is not None:
raise ValueError('*timeout* argument is not supported')
hit = re.match(r'^(https?)://([^:/]+)(?::(\d+))?(.*)$', url)
if not hit:
raise ValueError('Unsupported URL: ' + url)
if hit.group(1) == 'https':
ssl_context = self.ssl_context
proxy = self.ssl_proxy
else:
ssl_context = None
proxy = self.proxy
hostname = hit.group(2)
if hit.group(3):
port = int(hit.group(3))
elif ssl_context:
port = 443
else:
port = 80
path = hit.group(4)
try:
conn = self.conn_pool[(hostname, port)]
except KeyError:
conn = HTTPConnection(hostname, port, proxy=proxy,
ssl_context=ssl_context)
self.conn_pool[(hostname, port)] = conn
try:
conn.send_request(method, path, headers=headers, body=body)
resp = conn.read_response()
except (dugong.ConnectionClosed, dugong.InvalidResponse, dugong.UnsupportedResponse,
dugong.ConnectionTimedOut, dugong.HostnameNotResolvable,
dugong.DNSUnavailable, ssl.SSLError) as exc:
raise g_auth.exceptions.TransportError(exc)
return Namespace(status=resp.status, headers=resp.headers,
data=conn.readall())
def test_dns_one(monkeypatch):
monkeypatch.setattr(dugong, 'DNS_TEST_HOSTNAMES',
(('localhost', 80),))
with pytest.raises(dugong.HostnameNotResolvable):
conn = HTTPConnection('foobar.invalid')
conn.connect()
class Backend(AbstractBackend, metaclass=ABCDocstMeta):
needs_login = True
known_options = {'test-string', 'ssl-ca-path'}
authorize_url = "/b2api/v1/b2_authorize_account"
authorize_hostname = "api.backblaze.com"
hdr_prefix = 'X-Bz-Info-'
# no chunksize limit on backend, so we try to limit the number of meta headers
hdr_chunksize = 2000
_add_meta_headers_s3 = s3c.Backend._add_meta_headers
_extractmeta_s3 = s3c.Backend._extractmeta
def __init__(self, storage_url, login, password, options):
super().__init__()
self.options = options
self.bucket_name = None
self.prefix = None
self.auth_token = None
self.api_host = None
self.download_host = None
self.conn_api = None
self.conn_download = None
self.account_id = login
self.account_key = password
self.bucket_id = None
self.bucket_name = None
self._parse_storage_url(storage_url)
# Add test header povided to all requests.
if options.get('test-string'):
# sanitize entry string
self.test_string = ''.join([
x for x in options.get('test-string')
if x in 'abcdefghijklmnopqrstuvwxyz_'
])
else:
self.test_string = None
self.ssl_context = get_ssl_context(options.get('ssl-ca-path', None))
self._login()
self._connect_bucket(self.bucket_name)
def _do_request(self,
method,
path,
conn,
headers=None,
body=None,
auth_token=None,
download_body=True,
body_size=None):
"""Send request, read and return response object
This method modifies the *headers* dictionary.
conn must by a HTTPConnection
When download_body is True, need to receive data before making new connection
"""
def _debug_body(b):
if isinstance(b, str):
return b
elif b is None:
return "None"
else:
return 'byte_body'
def _debug_hostname(c):
try:
return c.hostname
except:
return "None"
log.debug('started with %r, %r, %r, %r, %r', method,
_debug_hostname(conn), path, headers, _debug_body(body))
if headers is None:
headers = CaseInsensitiveDict()
if auth_token is None:
headers['Authorization'] = self.auth_token
else:
headers['Authorization'] = auth_token
if self.test_string:
headers['X-Bz-Test-Mode'] = self.test_string
try:
if isinstance(body, io.FileIO):
if body_size is None:
raise ValueError(
"Body size is necessary when uploading from file")
conn.send_request(method,
path,
headers=headers,
body=BodyFollowing(body_size))
while True:
buf = body.read(BUFSIZE)
if not buf:
break
conn.write(buf)
else:
conn.send_request(method, path, headers=headers, body=body)
resp = conn.read_response()
if download_body or resp.status != 200:
body = conn.readall()
else:
# caller need to download body itself before making new request
body = None
except Exception as exc:
if is_temp_network_error(exc):
# We probably can't use the connection anymore
conn.disconnect()
raise
if resp.status == 200 or resp.status == 206:
return resp, body
try:
# error code is in body
j = json.loads(str(body, encoding='UTF-8'))
except ValueError:
raise HTTPError(resp.status, resp.reason, resp.headers)
# Expired auth token
if resp.status == 401:
if j['code'] == 'expired_auth_token':
log.info(
'BackBlaze auth token seems to have expired, requesting new one.'
)
self.conn_api.disconnect()
self.conn_download.disconnect()
# Force constructing a new connection with a new token, otherwise
# the connection will be reestablished with the same token.
self.conn_api = None
self.conn_download = None
self._login()
raise AuthenticationExpired(j['message'])
else:
raise AuthorizationError(j['message'])
# File not found
if resp.status == 404:
raise NoSuchObject(path)
# Backend error
raise B2Error(j['status'], j['code'], j['message'], headers=headers)
def _login(self):
"""
Login with backend and make a new connection to API and download server
"""
id_and_key = self.account_id + ':' + self.account_key
basic_auth_string = 'Basic ' + str(
base64.b64encode(bytes(id_and_key, 'UTF-8')), encoding='UTF-8')
with HTTPConnection(self.authorize_hostname,
443,
ssl_context=self.ssl_context) as conn:
resp, body = self._do_request('GET',
self.authorize_url,
conn,
auth_token=basic_auth_string)
j = json.loads(str(body, encoding='UTF-8'))
api_url = urllib.parse.urlparse(j['apiUrl'])
download_url = urllib.parse.urlparse(j['downloadUrl'])
self.api_host = api_url.hostname
self.auth_token = j['authorizationToken']
self.download_host = download_url.hostname
self.conn_api = HTTPConnection(self.api_host,
443,
ssl_context=self.ssl_context)
self.conn_download = HTTPConnection(self.download_host,
443,
ssl_context=self.ssl_context)
def _connect_bucket(self, bucket_name):
"""
Get id of bucket_name
"""
log.debug('started with %s' % (bucket_name))
resp, body = self._do_request(
'GET', '/b2api/v1/b2_list_buckets?accountId=%s' % self.account_id,
self.conn_api)
bucket_id = None
j = json.loads(str(body, encoding='UTF-8'))
for b in j['buckets']:
if b['bucketName'] == bucket_name:
bucket_id = b['bucketId']
if bucket_id is None:
raise DanglingStorageURLError(bucket_name)
self.bucket_id = bucket_id
self.bucket_name = bucket_name
def _add_meta_headers(self, headers, metadata):
self._add_meta_headers_s3(headers,
metadata,
chunksize=self.hdr_chunksize)
# URL encode headers
for i in count():
# Headers is an email.message object, so indexing it
# would also give None instead of KeyError
key = '%smeta-%03d' % (self.hdr_prefix, i)
part = headers.get(key, None)
if part is None:
break
headers[key] = urllib.parse.quote(part.encode('utf-8'))
# Check we dont reach the metadata backend lmits
if i > 10:
raise RuntimeError("Too metadata for the backend")
def _extractmeta(self, resp, filename):
# URL decode headers
for i in count():
# Headers is an email.message object, so indexing it
# would also give None instead of KeyError
key = '%smeta-%03d' % (self.hdr_prefix, i)
part = resp.headers.get(key, None)
if part is None:
break
resp.headers.replace_header(key,
urllib.parse.unquote_plus(str(part)))
return self._extractmeta_s3(resp, filename)
def _get_upload_url(self):
"""Get a single use URL to upload a file"""
log.debug('started')
body = bytes(json.dumps({'bucketId': self.bucket_id}),
encoding='UTF-8')
resp, body = self._do_request('POST',
'/b2api/v1/b2_get_upload_url',
self.conn_api,
body=body)
j = json.loads(str(body, encoding='UTF-8'))
return j['authorizationToken'], j['uploadUrl']
def _parse_storage_url(self, storage_url):
"""Init instance variables from storage url"""
hit = re.match(r'^b2?://([^/]+)(?:/(.*))?$', storage_url)
if not hit:
raise QuietError('Invalid storage URL', exitcode=2)
bucket_name = hit.group(1)
if not re.match('^(?!b2-)[a-z0-9A-Z\-]{6,50}$', bucket_name):
raise QuietError('Invalid bucket name.', exitcode=2)
prefix = hit.group(2) or ''
#remove trailing slash if exist
prefix = prefix[:-1] if prefix[-1] == '/' else prefix
self.bucket_name = bucket_name
self.prefix = prefix
def _delete_file_id(self, filelist):
for file in filelist:
body = bytes(json.dumps({
'fileName': file['fileName'],
'fileId': file['fileId']
}),
encoding='UTF-8')
log.debug('started with /file/%s/%s/%s' %
(self.bucket_name, self.prefix, file['fileName']))
try:
self._do_request('POST',
'/b2api/v1/b2_delete_file_version',
self.conn_api,
body=body)
except B2Error as err:
# Server may return file_not_present
# just let it close connection and retry
if err.code == 'file_not_present':
pass
else:
raise err
@retry
def _list_file_version(self, key, max_filecount=1000):
if max_filecount > 1000:
raise ValueError('max_filecount maximum is 1000')
request_dict = dict(bucketId=self.bucket_id,
maxFileCount=max_filecount)
request_dict['startFileName'] = self.prefix + '/' + self._encode_key(
key)
# supposing there is less than 1000 old file version
body = bytes(json.dumps(request_dict), encoding='UTF-8')
resp, body = self._do_request('POST',
'/b2api/v1/b2_list_file_versions',
self.conn_api,
body=body)
j = json.loads(str(body, encoding='UTF-8'))
r = []
# We suppose there is less than 1000 file version
for f in j['files']:
if self._decode_key(f['fileName']) == self.prefix + '/' + key:
r.append({'fileName': f['fileName'], 'fileId': f['fileId']})
return r
@retry
def _list_file_name(self, start_filename=None, max_filecount=1000):
if max_filecount > 1000:
raise ValueError('max_filecount maximum is 1000')
request_dict = {
'bucketId': self.bucket_id,
'maxFileCount': max_filecount
}
if start_filename is not None:
request_dict['startFileName'] = start_filename
body = bytes(json.dumps(request_dict), encoding='UTF-8')
resp, body = self._do_request('POST',
'/b2api/v1/b2_list_file_names',
self.conn_api,
body=body)
j = json.loads(str(body, encoding='UTF-8'))
filelist = [f['fileName'] for f in j['files']]
return j['nextFileName'], filelist
def _encode_key(self, filename):
# URLencode filename
filename = urllib.parse.quote(filename.encode('utf-8'), safe='/\\')
# DIRTY HACK :
# Backend does not support backslashes, we change them to pass test
filename = filename.replace("\\", "__")
return filename
def _decode_key(self, filename):
# DIRTY HACK :
# Backend does not support backslashes, we change them to pass test
filename = filename.replace("__", "\\")
# URLencode filename
filename = urllib.parse.unquote(filename)
return filename
def has_native_rename(self):
"""True if the backend has a native, atomic rename operation"""
return False
@copy_ancestor_docstring
def is_temp_failure(self, exc):
if isinstance(exc, AuthenticationExpired):
return True
if isinstance(exc, ssl.SSLError):
return True
if is_temp_network_error(exc):
return True
if isinstance(exc, HTTPError):
return True
if isinstance(exc, B2Error):
if (exc.status == 400 or \
exc.status == 408 or \
(exc.status >= 500 and exc.status <= 599)):
return True
return False
@retry
@copy_ancestor_docstring
def lookup(self, key):
log.debug('started with %s', key)
key = self._encode_key(key)
headers = CaseInsensitiveDict()
headers['Range'] = "bytes=0-1" # Only get first byte
resp, data = self._do_request('GET',
'/file/%s/%s/%s' %
(self.bucket_name, self.prefix, key),
self.conn_download,
headers=headers)
meta = self._extractmeta(resp, key)
return meta
@retry
def get_size(self, key):
"""Return size of object stored under *key*"""
log.debug('started with %s', key)
key = self._encode_key(key)
key_with_prefix = "%s/%s" % (self.prefix, key)
request_dict = {
'bucketId': self.bucket_id,
'startFileName': key_with_prefix,
'maxFileCount': 1
}
body = bytes(json.dumps(request_dict), encoding='UTF-8')
resp, body = self._do_request('POST',
'/b2api/v1/b2_list_file_names',
self.conn_api,
body=body)
j = json.loads(str(body, encoding='UTF-8'))
if j['files'][0]['fileName'] == key_with_prefix:
return j['files'][0]['contentLength']
raise NoSuchObject(key_with_prefix)
@retry
@copy_ancestor_docstring
def open_read(self, key):
log.debug('started with %s', key)
key = self._encode_key(key)
resp, data = self._do_request('GET',
'/file/%s/%s/%s' %
(self.bucket_name, self.prefix, key),
self.conn_download,
download_body=False)
meta = self._extractmeta(resp, key)
return ObjectR(key, self.conn_download, resp, metadata=meta)
@copy_ancestor_docstring
def open_write(self, key, metadata=None, is_compressed=False):
log.debug('started with %s', key)
key = self._encode_key(key)
return ObjectW(key, self, metadata)
@retry
@copy_ancestor_docstring
def clear(self):
log.debug('started')
for file in self.list():
self.delete(file, force=True)
@retry
@copy_ancestor_docstring
def delete(self, key, force=False):
log.debug('started with %s', key)
todel_id = self._list_file_version(key)
if not todel_id and not force:
raise NoSuchObject(key)
self._delete_file_id(todel_id)
@copy_ancestor_docstring
def list(self, prefix=''):
log.debug('started with %s', prefix)
prefix = self._encode_key(prefix)
next_filename = self.prefix + '/' + prefix
keys_remaining = True
while keys_remaining and next_filename is not None:
next_filename, filelist = self._list_file_name(next_filename)
while filelist:
file = filelist.pop(0)
if file.startswith(self.prefix + '/' + prefix):
# remove prefix before return
r = file[len(self.prefix + '/'):]
yield self._decode_key(r)
else:
keys_remaining = False
break
@prepend_ancestor_docstring
def copy(self, src, dest, metadata=None):
"""No atomic copy operation on backend. Hope this does not append to often"""
log.debug('started with %s, %s', src, dest)
data, src_meta = self.fetch(src)
# Delete dest file if already exist
self.delete(dest, force=True)
if metadata is None:
dst_meta = src_meta
else:
dst_meta = metadata
self.store(dest, data, dst_meta)
@copy_ancestor_docstring
def update_meta(self, key, metadata):
log.debug('started with %s', key)
self.copy(key, key, metadata)
def close(self):
log.debug('started')
self.conn_api.disconnect()
self.conn_download.disconnect()
def test_dns_two(monkeypatch):
monkeypatch.setattr(dugong, 'DNS_TEST_HOSTNAMES',
(('grumpf.invalid', 80),))
with pytest.raises(dugong.DNSUnavailable):
conn = HTTPConnection('foobar.invalid')
conn.connect()
raise SystemExit('Can only do http')
path_list.append(urlunsplit(('', '') + o[2:4] + ('',)))
# Code from here on is included in documentation
# start-example
import asyncio
import atexit
from dugong import HTTPConnection, AioFuture
# Get a MainLoop instance from the asyncio module to switch
# between coroutines (and clean up at program exit)
loop = asyncio.get_event_loop()
atexit.register(loop.close)
with HTTPConnection(hostname, port) as conn:
# This generator function returns a coroutine that sends
# all the requests.
def send_requests():
for path in path_list:
yield from conn.co_send_request('GET', path)
# This generator function returns a coroutine that reads
# all the responses
def read_responses():
bodies = []
for path in path_list:
resp = yield from conn.co_read_response()
assert resp.status == 200
buf = yield from conn.co_readall()
bodies.append(buf)
def _detect_features(self, hostname, port, ssl_context):
'''Try to figure out the Swift version and supported features by
examining the /info endpoint of the storage server.
See https://docs.openstack.org/swift/latest/middleware.html#discoverability
'''
if 'no-feature-detection' in self.options:
log.debug('Skip feature detection')
return
if not port:
port = 443 if ssl_context else 80
detected_features = Features()
with HTTPConnection(hostname, port, proxy=self.proxy,
ssl_context=ssl_context) as conn:
conn.timeout = int(self.options.get('tcp-timeout', 20))
log.debug('GET /info')
conn.send_request('GET', '/info')
resp = conn.read_response()
# 200, 401, 403 and 404 are all OK since the /info endpoint
# may not be accessible (misconfiguration) or may not
# exist (old Swift version).
if resp.status not in (200, 401, 403, 404):
log.error("Wrong server response.\n%s",
self._dump_response(resp, body=conn.read(2048)))
raise HTTPError(resp.status, resp.reason, resp.headers)
if resp.status == 200:
hit = re.match(r'^application/json(;\s*charset="?(.+?)"?)?$',
resp.headers['content-type'])
if not hit:
log.error("Wrong server response. Expected json. Got: \n%s",
self._dump_response(resp, body=conn.read(2048)))
raise RuntimeError('Unexpected server reply')
info = json.loads(conn.readall().decode(hit.group(2) or 'utf-8'))
swift_info = info.get('swift', {})
log.debug('%s:%s/info returns %s', hostname, port, info)
swift_version_string = swift_info.get('version', None)
if swift_version_string and \
LooseVersion(swift_version_string) >= LooseVersion('2.8'):
detected_features.has_copy = True
# Default metadata value length constrain is 256 bytes
# but the provider could configure another value.
# We only decrease the chunk size since 255 is a big enough chunk size.
max_meta_len = swift_info.get('max_meta_value_length', None)
if isinstance(max_meta_len, int) and max_meta_len < 256:
detected_features.max_meta_len = max_meta_len
if info.get('bulk_delete', False):
detected_features.has_bulk_delete = True
bulk_delete = info['bulk_delete']
assert bulk_delete.get('max_failed_deletes', 1000) <= \
bulk_delete.get('max_deletes_per_request', 10000)
assert bulk_delete.get('max_failed_deletes', 1000) > 0
# The block cache removal queue has a capacity of 1000.
# We do not need bigger values than that.
# We use max_failed_deletes instead of max_deletes_per_request
# because then we can be sure even when all our delete requests
# get rejected we get a complete error list back from the server.
# If we would set the value higher, _delete_multi() would maybe
# delete some entries from the *keys* list that did not get
# deleted and would miss them in a retry.
detected_features.max_deletes = min(1000,
int(bulk_delete.get('max_failed_deletes', 1000)))
log.info('Detected Swift features for %s:%s: %s',
hostname, port, detected_features, extra=LOG_ONCE)
else:
log.debug('%s:%s/info not found or not accessible. Skip feature detection.',
hostname, port)
self.features = detected_features
def _get_conn(self):
'''Obtain connection to server and authentication token'''
log.debug('started')
if 'no-ssl' in self.options:
ssl_context = None
else:
ssl_context = self.ssl_context
headers = CaseInsensitiveDict()
headers['Content-Type'] = 'application/json'
headers['Accept'] = 'application/json; charset="utf-8"'
if ':' in self.login:
(tenant, user) = self.login.split(':')
else:
tenant = None
user = self.login
auth_body = {
'auth': {
'passwordCredentials': {
'username': user,
'password': self.password
}
}
}
if tenant:
auth_body['auth']['tenantName'] = tenant
with HTTPConnection(self.hostname,
port=self.port,
proxy=self.proxy,
ssl_context=ssl_context) as conn:
conn.timeout = int(self.options.get('tcp-timeout', 20))
conn.send_request('POST',
'/v2.0/tokens',
headers=headers,
body=json.dumps(auth_body).encode('utf-8'))
resp = conn.read_response()
if resp.status == 401:
raise AuthorizationError(resp.reason)
elif resp.status > 299 or resp.status < 200:
raise HTTPError(resp.status, resp.reason, resp.headers)
cat = json.loads(conn.read().decode('utf-8'))
self.auth_token = cat['access']['token']['id']
avail_regions = []
for service in cat['access']['serviceCatalog']:
if service['type'] != 'object-store':
continue
for endpoint in service['endpoints']:
if endpoint['region'] != self.region:
avail_regions.append(endpoint['region'])
continue
o = urlsplit(endpoint['publicURL'])
self.auth_prefix = urllib.parse.unquote(o.path)
if o.scheme == 'https':
ssl_context = self.ssl_context
elif o.scheme == 'http':
ssl_context = None
else:
# fall through to scheme used for authentication
pass
self._detect_features(o.hostname, o.port, ssl_context)
conn = HTTPConnection(o.hostname,
o.port,
proxy=self.proxy,
ssl_context=ssl_context)
conn.timeout = int(self.options.get('tcp-timeout', 20))
return conn
if len(avail_regions) < 10:
raise DanglingStorageURLError(
self.container_name,
'No accessible object storage service found in region %s'
' (available regions: %s)' %
(self.region, ', '.join(avail_regions)))
else:
raise DanglingStorageURLError(
self.container_name,
'No accessible object storage service found in region %s' %
self.region)
def _get_conn(self):
'''Obtain connection to server and authentication token'''
log.debug('started')
if 'no-ssl' in self.options:
ssl_context = None
else:
ssl_context = self.ssl_context
headers = CaseInsensitiveDict()
headers['Content-Type'] = 'application/json'
headers['Accept'] = 'application/json; charset="utf-8"'
if ':' in self.login:
(tenant,user) = self.login.split(':')
else:
tenant = None
user = self.login
auth_body = { 'auth':
{ 'passwordCredentials':
{ 'username': user,
'password': self.password } }}
if tenant:
auth_body['auth']['tenantName'] = tenant
with HTTPConnection(self.hostname, port=self.port, proxy=self.proxy,
ssl_context=ssl_context) as conn:
conn.timeout = int(self.options.get('tcp-timeout', 20))
conn.send_request('POST', '/v2.0/tokens', headers=headers,
body=json.dumps(auth_body).encode('utf-8'))
resp = conn.read_response()
if resp.status == 401:
raise AuthorizationError(resp.reason)
elif resp.status > 299 or resp.status < 200:
raise HTTPError(resp.status, resp.reason, resp.headers)
cat = json.loads(conn.read().decode('utf-8'))
self.auth_token = cat['access']['token']['id']
avail_regions = []
for service in cat['access']['serviceCatalog']:
if service['type'] != 'object-store':
continue
for endpoint in service['endpoints']:
if endpoint['region'] != self.region:
avail_regions.append(endpoint['region'])
continue
o = urlsplit(endpoint['publicURL'])
self.auth_prefix = urllib.parse.unquote(o.path)
if o.scheme == 'https':
ssl_context = self.ssl_context
elif o.scheme == 'http':
ssl_context = None
else:
# fall through to scheme used for authentication
pass
conn = HTTPConnection(o.hostname, o.port, proxy=self.proxy,
ssl_context=ssl_context)
conn.timeout = int(self.options.get('tcp-timeout', 20))
return conn
if len(avail_regions) < 10:
raise DanglingStorageURLError(self.container_name,
'No accessible object storage service found in region %s'
' (available regions: %s)' % (self.region, ', '.join(avail_regions)))
else:
raise DanglingStorageURLError(self.container_name,
'No accessible object storage service found in region %s'
% self.region)
def test_dns_one(monkeypatch):
monkeypatch.setattr(dugong, 'DNS_TEST_HOSTNAMES', (('localhost', 80), ))
with pytest.raises(dugong.HostnameNotResolvable):
conn = HTTPConnection('foobar.invalid')
conn.connect()
def test_dns_two(monkeypatch):
monkeypatch.setattr(dugong, 'DNS_TEST_HOSTNAMES',
(('grumpf.invalid', 80), ))
with pytest.raises(dugong.DNSUnavailable):
conn = HTTPConnection('foobar.invalid')
conn.connect()
def _get_conn(self):
'''Obtain connection to server and authentication token'''
log.debug('started')
if 'no-ssl' in self.options:
ssl_context = None
else:
ssl_context = self.ssl_context
headers = CaseInsensitiveDict()
headers['Content-Type'] = 'application/json'
headers['Accept'] = 'application/json; charset="utf-8"'
if ':' in self.login:
(tenant,user) = self.login.split(':')
else:
tenant = None
user = self.login
domain = self.options.get('domain', None)
if domain:
if not tenant:
raise ValueError("Tenant is required when Keystone v3 is used")
# In simple cases where there's only one domain, the project domain
# will be the same as the authentication domain, but this option
# allows for them to be different
project_domain = self.options.get('project-domain', domain)
auth_body = {
'auth': {
'identity': {
'methods': ['password'],
'password': {
'user': {
'name': user,
'domain': {
'id': domain
},
'password': self.password
}
}
},
'scope': {
'project': {
'id': tenant,
'domain': {
'id': project_domain
}
}
}
}
}
auth_url_path = '/v3/auth/tokens'
else:
# If a domain is not specified, assume v2
auth_body = { 'auth':
{ 'passwordCredentials':
{ 'username': user,
'password': self.password } }}
auth_url_path = '/v2.0/tokens'
if tenant:
auth_body['auth']['tenantName'] = tenant
with HTTPConnection(self.hostname, port=self.port, proxy=self.proxy,
ssl_context=ssl_context) as conn:
conn.timeout = int(self.options.get('tcp-timeout', 20))
conn.send_request('POST', auth_url_path, headers=headers,
body=json.dumps(auth_body).encode('utf-8'))
resp = conn.read_response()
if resp.status == 401:
raise AuthorizationError(resp.reason)
elif resp.status > 299 or resp.status < 200:
raise HTTPError(resp.status, resp.reason, resp.headers)
cat = json.loads(conn.read().decode('utf-8'))
if self.options.get('domain', None):
self.auth_token = resp.headers['X-Subject-Token']
service_catalog = cat['token']['catalog']
else:
self.auth_token = cat['access']['token']['id']
service_catalog = cat['access']['serviceCatalog']
avail_regions = []
for service in service_catalog:
if service['type'] != 'object-store':
continue
for endpoint in service['endpoints']:
if endpoint['region'] != self.region:
avail_regions.append(endpoint['region'])
continue
if 'publicURL' in endpoint:
# The publicURL nomenclature is found in v2 catalogs
o = urlsplit(endpoint['publicURL'])
else:
# Whereas v3 catalogs do 'interface' == 'public' and
# 'url' for the URL itself
if endpoint['interface'] != 'public':
continue
o = urlsplit(endpoint['url'])
self.auth_prefix = urllib.parse.unquote(o.path)
if o.scheme == 'https':
ssl_context = self.ssl_context
elif o.scheme == 'http':
ssl_context = None
else:
# fall through to scheme used for authentication
pass
self._detect_features(o.hostname, o.port, ssl_context)
conn = HTTPConnection(o.hostname, o.port, proxy=self.proxy,
ssl_context=ssl_context)
conn.timeout = int(self.options.get('tcp-timeout', 20))
return conn
if len(avail_regions) < 10:
raise DanglingStorageURLError(self.container_name,
'No accessible object storage service found in region %s'
' (available regions: %s)' % (self.region, ', '.join(avail_regions)))
else:
raise DanglingStorageURLError(self.container_name,
'No accessible object storage service found in region %s'
% self.region)
# When running from HG repo, enable all warnings
if os.path.exists(os.path.join(basedir, '.hg')):
import warnings
warnings.simplefilter('error')
from dugong import HTTPConnection, BUFFER_SIZE
for arg in sys.argv[1:]:
url = urlsplit(arg)
assert url.scheme == 'http'
path = url.path
if url.query:
path += '?' + url.query
with HTTPConnection(url.hostname, url.port) as conn:
conn.send_request('GET', path)
resp = conn.read_response()
if resp.status != 200:
raise SystemExit('%d %s' % (resp.status, resp.reason))
# Determine if we're reading text or binary data, and (in case of text),
# what character set is being used.
if 'Content-Type' not in resp.headers:
type_ = 'application/octet-stream'
else:
type_ = resp.headers['Content-Type']
hit = re.match(r'(.+?)(?:; charset=(.+))?$', type_)
if not hit:
raise SystemExit('Unable to parse content-type: %s' % type_)
