def _get_conn(self): '''Return connection to server''' conn = HTTPConnection(self.hostname, self.port, proxy=self.proxy, ssl_context=self.ssl_context) conn.timeout = int(self.options.get('tcp-timeout', 20)) return conn
def _login(self): """ Login with backend and make a new connection to API and download server """ id_and_key = self.account_id + ':' + self.account_key basic_auth_string = 'Basic ' + str( base64.b64encode(bytes(id_and_key, 'UTF-8')), encoding='UTF-8') with HTTPConnection(self.authorize_hostname, 443, ssl_context=self.ssl_context) as conn: resp, body = self._do_request('GET', self.authorize_url, conn, auth_token=basic_auth_string) j = json.loads(str(body, encoding='UTF-8')) api_url = urllib.parse.urlparse(j['apiUrl']) download_url = urllib.parse.urlparse(j['downloadUrl']) self.api_host = api_url.hostname self.auth_token = j['authorizationToken'] self.download_host = download_url.hostname self.conn_api = HTTPConnection(self.api_host, 443, ssl_context=self.ssl_context) self.conn_download = HTTPConnection(self.download_host, 443, ssl_context=self.ssl_context)
def _get_conn(self): '''Obtain connection to server and authentication token''' log.debug('started') if 'no-ssl' in self.options: ssl_context = None else: ssl_context = self.ssl_context headers = CaseInsensitiveDict() headers['X-Auth-User'] = self.login headers['X-Auth-Key'] = self.password with HTTPConnection(self.hostname, self.port, proxy=self.proxy, ssl_context=ssl_context) as conn: conn.timeout = int(self.options.get('tcp-timeout', 20)) for auth_path in ('/v1.0', '/auth/v1.0'): log.debug('GET %s', auth_path) conn.send_request('GET', auth_path, headers=headers) resp = conn.read_response() if resp.status in (404, 412): log.debug('auth to %s failed, trying next path', auth_path) conn.discard() continue elif resp.status == 401: raise AuthorizationError(resp.reason) elif resp.status > 299 or resp.status < 200: raise HTTPError(resp.status, resp.reason, resp.headers) # Pylint can't infer SplitResult Types #pylint: disable=E1103 self.auth_token = resp.headers['X-Auth-Token'] o = urlsplit(resp.headers['X-Storage-Url']) self.auth_prefix = urllib.parse.unquote(o.path) if o.scheme == 'https': ssl_context = self.ssl_context elif o.scheme == 'http': ssl_context = None else: # fall through to scheme used for authentication pass conn = HTTPConnection(o.hostname, o.port, proxy=self.proxy, ssl_context=ssl_context) conn.timeout = int(self.options.get('tcp-timeout', 20)) return conn raise RuntimeError('No valid authentication path found')
def test_http_proxy(http_server, monkeypatch, test_port): test_host = 'www.foobarz.invalid' test_path = '/someurl?barf' get_path = None def do_GET(self): nonlocal get_path get_path = self.path self.send_response(200) self.send_header("Content-Type", 'application/octet-stream') self.send_header("Content-Length", '0') self.end_headers() monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET) conn = HTTPConnection(test_host, test_port, proxy=(http_server.host, http_server.port)) try: conn.send_request('GET', test_path) resp = conn.read_response() assert resp.status == 200 conn.discard() finally: conn.disconnect() if test_port is None: exp_path = 'http://%s%s' % (test_host, test_path) else: exp_path = 'http://%s:%d%s' % (test_host, test_port, test_path) assert get_path == exp_path
def _get_conn(self): '''Obtain connection to server and authentication token''' log.debug('started') if 'no-ssl' in self.options: ssl_context = None else: ssl_context = self.ssl_context headers = CaseInsensitiveDict() headers['X-Auth-User'] = self.login headers['X-Auth-Key'] = self.password with HTTPConnection(self.hostname, self.port, proxy=self.proxy, ssl_context=ssl_context) as conn: conn.timeout = int(self.options.get('tcp-timeout', 10)) for auth_path in ('/v1.0', '/auth/v1.0'): log.debug('GET %s', auth_path) conn.send_request('GET', auth_path, headers=headers) resp = conn.read_response() if resp.status in (404, 412): log.debug('auth to %s failed, trying next path', auth_path) conn.discard() continue elif resp.status == 401: raise AuthorizationError(resp.reason) elif resp.status > 299 or resp.status < 200: raise HTTPError(resp.status, resp.reason, resp.headers) # Pylint can't infer SplitResult Types #pylint: disable=E1103 self.auth_token = resp.headers['X-Auth-Token'] o = urlsplit(resp.headers['X-Storage-Url']) self.auth_prefix = urllib.parse.unquote(o.path) if o.scheme == 'https': ssl_context = self.ssl_context elif o.scheme == 'http': ssl_context = None else: # fall through to scheme used for authentication pass conn = HTTPConnection(o.hostname, o.port, proxy=self.proxy, ssl_context=ssl_context) conn.timeout = int(self.options.get('tcp-timeout', 10)) return conn raise RuntimeError('No valid authentication path found')
def test_invalid_ssl(): check_http_connection() # Don't load certificates context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context.options |= ssl.OP_NO_SSLv2 context.verify_mode = ssl.CERT_REQUIRED conn = HTTPConnection(SSL_TEST_HOST, ssl_context=context) with pytest.raises(ssl.SSLError): conn.send_request('GET', '/') conn.disconnect()
def close(self): """Close object and upload data""" log.debug('started with %s', self.key) if self.closed: # still call fh.close, may have generated an error before self.fh.close() return self.fh.seek(0) upload_auth_token, upload_url = self.backend._get_upload_url() upload_url = urllib.parse.urlparse(upload_url) with HTTPConnection(upload_url.hostname, 443, ssl_context=self.backend.ssl_context) as conn_up: headers = CaseInsensitiveDict() headers['X-Bz-File-Name'] = self.backend.prefix + self.key headers['Content-Type'] = 'application/octet-stream' headers['Content-Length'] = self.obj_size headers['X-Bz-Content-Sha1'] = self.sha1.hexdigest() if self.meta is None: self.meta = dict() self.backend._add_meta_headers(headers, self.meta) self.backend._do_request('POST', upload_url.path + '?' + upload_url.query, conn_up, headers=headers, body=self.fh, auth_token=upload_auth_token, body_size=self.obj_size) self.fh.close() self.closed = True
def test_connect_proxy(http_server, monkeypatch, test_port): test_host = 'www.foobarz.invalid' test_path = '/someurl?barf' connect_path = None def do_CONNECT(self): # Pretend we're the remote server too nonlocal connect_path connect_path = self.path self.send_response(200) self.end_headers() self.close_connection = 0 monkeypatch.setattr(MockRequestHandler, 'do_CONNECT', do_CONNECT, raising=False) get_path = None def do_GET(self): nonlocal get_path get_path = self.path self.send_response(200) self.send_header("Content-Type", 'application/octet-stream') self.send_header("Content-Length", '0') self.end_headers() monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET) # We don't *actually* want to establish SSL, that'd be # to complex for our mock server monkeypatch.setattr('ssl.match_hostname', lambda x, y: True) conn = HTTPConnection(test_host, test_port, proxy=(http_server.host, http_server.port), ssl_context=FakeSSLContext()) try: conn.send_request('GET', test_path) resp = conn.read_response() assert resp.status == 200 conn.discard() finally: conn.disconnect() if test_port is None: test_port = 443 exp_path = '%s:%d' % (test_host, test_port) assert connect_path == exp_path assert get_path == test_path
def _get_access_token(self): log.info('Requesting new access token') headers = CaseInsensitiveDict() headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=utf-8' body = urllib.parse.urlencode({ 'client_id': OAUTH_CLIENT_ID, 'client_secret': OAUTH_CLIENT_SECRET, 'refresh_token': self.password, 'grant_type': 'refresh_token' }) conn = HTTPConnection('accounts.google.com', 443, proxy=self.proxy, ssl_context=self.ssl_context) try: conn.send_request('POST', '/o/oauth2/token', headers=headers, body=body.encode('utf-8')) resp = conn.read_response() json_resp = self._parse_json_response(resp, conn) if resp.status > 299 or resp.status < 200: assert 'error' in json_resp if 'error' in json_resp: raise AuthenticationError(json_resp['error']) else: self.access_token[self.password] = json_resp['access_token'] finally: conn.disconnect()
def test_connect_ssl(): check_http_connection() ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) ssl_context.options |= ssl.OP_NO_SSLv2 ssl_context.verify_mode = ssl.CERT_REQUIRED ssl_context.set_default_verify_paths() conn = HTTPConnection(SSL_TEST_HOST, ssl_context=ssl_context) conn.send_request('GET', '/') resp = conn.read_response() assert resp.status in (200, 301, 302) assert resp.path == '/' conn.discard() conn.disconnect()
def _get_download_connection(self): if self.download_url is None: self._authorize_account() if self.download_connection is None: self.download_connection = HTTPConnection( self.download_url.hostname, 443, ssl_context=self.ssl_context) self.download_connection.timeout = self.tcp_timeout return self.download_connection
def _request_upload_url_info(self): request_data = {'bucketId': self._get_bucket_id()} response = self._do_api_call('b2_get_upload_url', request_data) new_upload_url = urlparse(response['uploadUrl']) new_authorization_token = response['authorizationToken'] upload_connection = HTTPConnection(new_upload_url.hostname, 443, ssl_context=self.ssl_context) upload_connection.timeout = self.tcp_timeout return { 'hostname': new_upload_url.hostname, 'connection': upload_connection, 'path': new_upload_url.path, 'authorizationToken': new_authorization_token, 'isUploading': False }
def _get_access_token(self): log.info('Requesting new access token') headers = CaseInsensitiveDict() headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=utf-8' body = urlencode({'client_id': OAUTH_CLIENT_ID, 'client_secret': OAUTH_CLIENT_SECRET, 'refresh_token': self.password, 'grant_type': 'refresh_token' }) conn = HTTPConnection('accounts.google.com', 443, proxy=self.proxy, ssl_context=self.ssl_context) try: conn.send_request('POST', '/o/oauth2/token', headers=headers, body=body.encode('utf-8')) resp = conn.read_response() if resp.status > 299 or resp.status < 200: raise HTTPError(resp.status, resp.reason, resp.headers) content_type = resp.headers.get('Content-Type', None) if content_type: hit = re.match(r'application/json(?:; charset="(.+)")?$', resp.headers['Content-Type'], re.IGNORECASE) else: hit = None if not hit: log.error('Unexpected server reply when refreshing access token:\n%s', self._dump_response(resp)) raise RuntimeError('Unable to parse server response') charset = hit.group(1) or 'utf-8' body = conn.readall().decode(charset) resp_json = json.loads(body) if not isinstance(resp_json, dict): log.error('Invalid json server response. Expected dict, got:\n%s', body) raise RuntimeError('Unable to parse server response') if 'error' in resp_json: raise AuthenticationError(resp_json['error']) if 'access_token' not in resp_json: log.error('Unable to find access token in server response:\n%s', body) raise RuntimeError('Unable to parse server response') self.access_token[self.password] = resp_json['access_token'] finally: conn.disconnect()
def main(): if len(sys.argv) != 2: raise SystemExit('Usage: %s <url>' % sys.argv[0]) url = sys.argv[1] url_els = urlsplit(url) if url_els.scheme == 'https': ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) ssl_context.options |= ssl.OP_NO_SSLv2 ssl_context.verify_mode = ssl.CERT_REQUIRED ssl_context.set_default_verify_paths() else: ssl_context = None with HTTPConnection(url_els.hostname, port=url_els.port, ssl_context=ssl_context) as conn: path = urlunsplit(('', '') + url_els[2:4] + ('', )) or '/' conn.send_request('GET', path) resp = conn.read_response() if resp.status != 200: raise SystemExit('%d %s' % (resp.status, resp.reason)) # Determine if we're reading text or binary data, and (in case of text), # what character set is being used. if 'Content-Type' not in resp.headers: type_ = 'application/octet-stream' else: type_ = resp.headers['Content-Type'] hit = re.match(r'text/x?html(?:; charset=(.+))?$', type_) if not hit: raise SystemExit('Server did not send html but %s' % type_) if hit.group(1): charset = hit.group(1) else: charset = 'latin1' html_stream = TextIOWrapper(conn, encoding=charset) parser = LinkExtractor() while True: buf = html_stream.read(16 * 1024) if not buf: break parser.feed(buf) for link in parser.links: print(urljoin(url, link))
def conn(request, http_server): if http_server.use_ssl: ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) ssl_context.options |= ssl.OP_NO_SSLv2 ssl_context.verify_mode = ssl.CERT_REQUIRED ssl_context.load_verify_locations( cafile=os.path.join(TEST_DIR, 'ca.crt')) else: ssl_context = None conn = HTTPConnection(http_server.host, port=http_server.port, ssl_context=ssl_context) request.addfinalizer(conn.disconnect) return conn
def test_connect_proxy(http_server, monkeypatch, test_port): test_host = 'www.foobarz.invalid' test_path = '/someurl?barf' connect_path = None def do_CONNECT(self): # Pretend we're the remote server too nonlocal connect_path connect_path = self.path self.send_response(200) self.end_headers() self.close_connection = 0 monkeypatch.setattr(MockRequestHandler, 'do_CONNECT', do_CONNECT, raising=False) get_path = None def do_GET(self): nonlocal get_path get_path = self.path self.send_response(200) self.send_header("Content-Type", 'application/octet-stream') self.send_header("Content-Length", '0') self.end_headers() monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET) # We don't *actually* want to establish SSL, that'd be # to complex for our mock server monkeypatch.setattr('ssl.match_hostname', lambda x,y: True) conn = HTTPConnection(test_host, test_port, proxy=(http_server.host, http_server.port), ssl_context=FakeSSLContext()) try: conn.send_request('GET', test_path) resp = conn.read_response() assert resp.status == 200 conn.discard() finally: conn.disconnect() if test_port is None: test_port = 443 exp_path = '%s:%d' % (test_host, test_port) assert connect_path == exp_path assert get_path == test_path
def _authorize_account(self): '''Authorize API calls''' authorize_host = 'api.backblazeb2.com' authorize_url = api_url_prefix + 'b2_authorize_account' id_and_key = self.b2_application_key_id + ':' + self.b2_application_key basic_auth_string = 'Basic ' + str( base64.b64encode(bytes(id_and_key, 'UTF-8')), encoding='UTF-8') with HTTPConnection(authorize_host, 443, ssl_context=self.ssl_context) as connection: headers = CaseInsensitiveDict() headers['Authorization'] = basic_auth_string connection.send_request('GET', authorize_url, headers=headers, body=None) response = connection.read_response() response_body = connection.readall() if response.status != 200: raise RuntimeError('Authorization failed.') j = json.loads(response_body.decode('utf-8')) self.account_id = j['accountId'] allowed_info = j.get('allowed') if allowed_info.get('bucketId'): self.bucket_id = allowed_info.get('bucketId') if allowed_info.get('bucketName') != self.bucket_name: raise RuntimeError( 'Provided API key can not access desired bucket.') if not self._check_key_capabilities(allowed_info): raise RuntimeError( 'Provided API key does not have the required capabilities.' ) self.api_url = urlparse(j['apiUrl']) self.download_url = urlparse(j['downloadUrl']) self.authorization_token = j['authorizationToken']
def __call__(self, url: str, method: str = 'GET', body=None, headers=None, timeout=None): # https://github.com/googleapis/google-auth-library-python/issues/318 if not isinstance(body, bytes): body = str(body).encode('ascii') if timeout is not None: raise ValueError('*timeout* argument is not supported') hit = re.match(r'^(https?)://([^:/]+)(?::(\d+))?(.*)$', url) if not hit: raise ValueError('Unsupported URL: ' + url) if hit.group(1) == 'https': ssl_context = self.ssl_context proxy = self.ssl_proxy else: ssl_context = None proxy = self.proxy hostname = hit.group(2) if hit.group(3): port = int(hit.group(3)) elif ssl_context: port = 443 else: port = 80 path = hit.group(4) try: conn = self.conn_pool[(hostname, port)] except KeyError: conn = HTTPConnection(hostname, port, proxy=proxy, ssl_context=ssl_context) self.conn_pool[(hostname, port)] = conn try: conn.send_request(method, path, headers=headers, body=body) resp = conn.read_response() except (dugong.ConnectionClosed, dugong.InvalidResponse, dugong.UnsupportedResponse, dugong.ConnectionTimedOut, dugong.HostnameNotResolvable, dugong.DNSUnavailable, ssl.SSLError) as exc: raise g_auth.exceptions.TransportError(exc) return Namespace(status=resp.status, headers=resp.headers, data=conn.readall())
def test_dns_one(monkeypatch): monkeypatch.setattr(dugong, 'DNS_TEST_HOSTNAMES', (('localhost', 80),)) with pytest.raises(dugong.HostnameNotResolvable): conn = HTTPConnection('foobar.invalid') conn.connect()
class Backend(AbstractBackend, metaclass=ABCDocstMeta): needs_login = True known_options = {'test-string', 'ssl-ca-path'} authorize_url = "/b2api/v1/b2_authorize_account" authorize_hostname = "api.backblaze.com" hdr_prefix = 'X-Bz-Info-' # no chunksize limit on backend, so we try to limit the number of meta headers hdr_chunksize = 2000 _add_meta_headers_s3 = s3c.Backend._add_meta_headers _extractmeta_s3 = s3c.Backend._extractmeta def __init__(self, storage_url, login, password, options): super().__init__() self.options = options self.bucket_name = None self.prefix = None self.auth_token = None self.api_host = None self.download_host = None self.conn_api = None self.conn_download = None self.account_id = login self.account_key = password self.bucket_id = None self.bucket_name = None self._parse_storage_url(storage_url) # Add test header povided to all requests. if options.get('test-string'): # sanitize entry string self.test_string = ''.join([ x for x in options.get('test-string') if x in 'abcdefghijklmnopqrstuvwxyz_' ]) else: self.test_string = None self.ssl_context = get_ssl_context(options.get('ssl-ca-path', None)) self._login() self._connect_bucket(self.bucket_name) def _do_request(self, method, path, conn, headers=None, body=None, auth_token=None, download_body=True, body_size=None): """Send request, read and return response object This method modifies the *headers* dictionary. conn must by a HTTPConnection When download_body is True, need to receive data before making new connection """ def _debug_body(b): if isinstance(b, str): return b elif b is None: return "None" else: return 'byte_body' def _debug_hostname(c): try: return c.hostname except: return "None" log.debug('started with %r, %r, %r, %r, %r', method, _debug_hostname(conn), path, headers, _debug_body(body)) if headers is None: headers = CaseInsensitiveDict() if auth_token is None: headers['Authorization'] = self.auth_token else: headers['Authorization'] = auth_token if self.test_string: headers['X-Bz-Test-Mode'] = self.test_string try: if isinstance(body, io.FileIO): if body_size is None: raise ValueError( "Body size is necessary when uploading from file") conn.send_request(method, path, headers=headers, body=BodyFollowing(body_size)) while True: buf = body.read(BUFSIZE) if not buf: break conn.write(buf) else: conn.send_request(method, path, headers=headers, body=body) resp = conn.read_response() if download_body or resp.status != 200: body = conn.readall() else: # caller need to download body itself before making new request body = None except Exception as exc: if is_temp_network_error(exc): # We probably can't use the connection anymore conn.disconnect() raise if resp.status == 200 or resp.status == 206: return resp, body try: # error code is in body j = json.loads(str(body, encoding='UTF-8')) except ValueError: raise HTTPError(resp.status, resp.reason, resp.headers) # Expired auth token if resp.status == 401: if j['code'] == 'expired_auth_token': log.info( 'BackBlaze auth token seems to have expired, requesting new one.' ) self.conn_api.disconnect() self.conn_download.disconnect() # Force constructing a new connection with a new token, otherwise # the connection will be reestablished with the same token. self.conn_api = None self.conn_download = None self._login() raise AuthenticationExpired(j['message']) else: raise AuthorizationError(j['message']) # File not found if resp.status == 404: raise NoSuchObject(path) # Backend error raise B2Error(j['status'], j['code'], j['message'], headers=headers) def _login(self): """ Login with backend and make a new connection to API and download server """ id_and_key = self.account_id + ':' + self.account_key basic_auth_string = 'Basic ' + str( base64.b64encode(bytes(id_and_key, 'UTF-8')), encoding='UTF-8') with HTTPConnection(self.authorize_hostname, 443, ssl_context=self.ssl_context) as conn: resp, body = self._do_request('GET', self.authorize_url, conn, auth_token=basic_auth_string) j = json.loads(str(body, encoding='UTF-8')) api_url = urllib.parse.urlparse(j['apiUrl']) download_url = urllib.parse.urlparse(j['downloadUrl']) self.api_host = api_url.hostname self.auth_token = j['authorizationToken'] self.download_host = download_url.hostname self.conn_api = HTTPConnection(self.api_host, 443, ssl_context=self.ssl_context) self.conn_download = HTTPConnection(self.download_host, 443, ssl_context=self.ssl_context) def _connect_bucket(self, bucket_name): """ Get id of bucket_name """ log.debug('started with %s' % (bucket_name)) resp, body = self._do_request( 'GET', '/b2api/v1/b2_list_buckets?accountId=%s' % self.account_id, self.conn_api) bucket_id = None j = json.loads(str(body, encoding='UTF-8')) for b in j['buckets']: if b['bucketName'] == bucket_name: bucket_id = b['bucketId'] if bucket_id is None: raise DanglingStorageURLError(bucket_name) self.bucket_id = bucket_id self.bucket_name = bucket_name def _add_meta_headers(self, headers, metadata): self._add_meta_headers_s3(headers, metadata, chunksize=self.hdr_chunksize) # URL encode headers for i in count(): # Headers is an email.message object, so indexing it # would also give None instead of KeyError key = '%smeta-%03d' % (self.hdr_prefix, i) part = headers.get(key, None) if part is None: break headers[key] = urllib.parse.quote(part.encode('utf-8')) # Check we dont reach the metadata backend lmits if i > 10: raise RuntimeError("Too metadata for the backend") def _extractmeta(self, resp, filename): # URL decode headers for i in count(): # Headers is an email.message object, so indexing it # would also give None instead of KeyError key = '%smeta-%03d' % (self.hdr_prefix, i) part = resp.headers.get(key, None) if part is None: break resp.headers.replace_header(key, urllib.parse.unquote_plus(str(part))) return self._extractmeta_s3(resp, filename) def _get_upload_url(self): """Get a single use URL to upload a file""" log.debug('started') body = bytes(json.dumps({'bucketId': self.bucket_id}), encoding='UTF-8') resp, body = self._do_request('POST', '/b2api/v1/b2_get_upload_url', self.conn_api, body=body) j = json.loads(str(body, encoding='UTF-8')) return j['authorizationToken'], j['uploadUrl'] def _parse_storage_url(self, storage_url): """Init instance variables from storage url""" hit = re.match(r'^b2?://([^/]+)(?:/(.*))?$', storage_url) if not hit: raise QuietError('Invalid storage URL', exitcode=2) bucket_name = hit.group(1) if not re.match('^(?!b2-)[a-z0-9A-Z\-]{6,50}$', bucket_name): raise QuietError('Invalid bucket name.', exitcode=2) prefix = hit.group(2) or '' #remove trailing slash if exist prefix = prefix[:-1] if prefix[-1] == '/' else prefix self.bucket_name = bucket_name self.prefix = prefix def _delete_file_id(self, filelist): for file in filelist: body = bytes(json.dumps({ 'fileName': file['fileName'], 'fileId': file['fileId'] }), encoding='UTF-8') log.debug('started with /file/%s/%s/%s' % (self.bucket_name, self.prefix, file['fileName'])) try: self._do_request('POST', '/b2api/v1/b2_delete_file_version', self.conn_api, body=body) except B2Error as err: # Server may return file_not_present # just let it close connection and retry if err.code == 'file_not_present': pass else: raise err @retry def _list_file_version(self, key, max_filecount=1000): if max_filecount > 1000: raise ValueError('max_filecount maximum is 1000') request_dict = dict(bucketId=self.bucket_id, maxFileCount=max_filecount) request_dict['startFileName'] = self.prefix + '/' + self._encode_key( key) # supposing there is less than 1000 old file version body = bytes(json.dumps(request_dict), encoding='UTF-8') resp, body = self._do_request('POST', '/b2api/v1/b2_list_file_versions', self.conn_api, body=body) j = json.loads(str(body, encoding='UTF-8')) r = [] # We suppose there is less than 1000 file version for f in j['files']: if self._decode_key(f['fileName']) == self.prefix + '/' + key: r.append({'fileName': f['fileName'], 'fileId': f['fileId']}) return r @retry def _list_file_name(self, start_filename=None, max_filecount=1000): if max_filecount > 1000: raise ValueError('max_filecount maximum is 1000') request_dict = { 'bucketId': self.bucket_id, 'maxFileCount': max_filecount } if start_filename is not None: request_dict['startFileName'] = start_filename body = bytes(json.dumps(request_dict), encoding='UTF-8') resp, body = self._do_request('POST', '/b2api/v1/b2_list_file_names', self.conn_api, body=body) j = json.loads(str(body, encoding='UTF-8')) filelist = [f['fileName'] for f in j['files']] return j['nextFileName'], filelist def _encode_key(self, filename): # URLencode filename filename = urllib.parse.quote(filename.encode('utf-8'), safe='/\\') # DIRTY HACK : # Backend does not support backslashes, we change them to pass test filename = filename.replace("\\", "__") return filename def _decode_key(self, filename): # DIRTY HACK : # Backend does not support backslashes, we change them to pass test filename = filename.replace("__", "\\") # URLencode filename filename = urllib.parse.unquote(filename) return filename def has_native_rename(self): """True if the backend has a native, atomic rename operation""" return False @copy_ancestor_docstring def is_temp_failure(self, exc): if isinstance(exc, AuthenticationExpired): return True if isinstance(exc, ssl.SSLError): return True if is_temp_network_error(exc): return True if isinstance(exc, HTTPError): return True if isinstance(exc, B2Error): if (exc.status == 400 or \ exc.status == 408 or \ (exc.status >= 500 and exc.status <= 599)): return True return False @retry @copy_ancestor_docstring def lookup(self, key): log.debug('started with %s', key) key = self._encode_key(key) headers = CaseInsensitiveDict() headers['Range'] = "bytes=0-1" # Only get first byte resp, data = self._do_request('GET', '/file/%s/%s/%s' % (self.bucket_name, self.prefix, key), self.conn_download, headers=headers) meta = self._extractmeta(resp, key) return meta @retry def get_size(self, key): """Return size of object stored under *key*""" log.debug('started with %s', key) key = self._encode_key(key) key_with_prefix = "%s/%s" % (self.prefix, key) request_dict = { 'bucketId': self.bucket_id, 'startFileName': key_with_prefix, 'maxFileCount': 1 } body = bytes(json.dumps(request_dict), encoding='UTF-8') resp, body = self._do_request('POST', '/b2api/v1/b2_list_file_names', self.conn_api, body=body) j = json.loads(str(body, encoding='UTF-8')) if j['files'][0]['fileName'] == key_with_prefix: return j['files'][0]['contentLength'] raise NoSuchObject(key_with_prefix) @retry @copy_ancestor_docstring def open_read(self, key): log.debug('started with %s', key) key = self._encode_key(key) resp, data = self._do_request('GET', '/file/%s/%s/%s' % (self.bucket_name, self.prefix, key), self.conn_download, download_body=False) meta = self._extractmeta(resp, key) return ObjectR(key, self.conn_download, resp, metadata=meta) @copy_ancestor_docstring def open_write(self, key, metadata=None, is_compressed=False): log.debug('started with %s', key) key = self._encode_key(key) return ObjectW(key, self, metadata) @retry @copy_ancestor_docstring def clear(self): log.debug('started') for file in self.list(): self.delete(file, force=True) @retry @copy_ancestor_docstring def delete(self, key, force=False): log.debug('started with %s', key) todel_id = self._list_file_version(key) if not todel_id and not force: raise NoSuchObject(key) self._delete_file_id(todel_id) @copy_ancestor_docstring def list(self, prefix=''): log.debug('started with %s', prefix) prefix = self._encode_key(prefix) next_filename = self.prefix + '/' + prefix keys_remaining = True while keys_remaining and next_filename is not None: next_filename, filelist = self._list_file_name(next_filename) while filelist: file = filelist.pop(0) if file.startswith(self.prefix + '/' + prefix): # remove prefix before return r = file[len(self.prefix + '/'):] yield self._decode_key(r) else: keys_remaining = False break @prepend_ancestor_docstring def copy(self, src, dest, metadata=None): """No atomic copy operation on backend. Hope this does not append to often""" log.debug('started with %s, %s', src, dest) data, src_meta = self.fetch(src) # Delete dest file if already exist self.delete(dest, force=True) if metadata is None: dst_meta = src_meta else: dst_meta = metadata self.store(dest, data, dst_meta) @copy_ancestor_docstring def update_meta(self, key, metadata): log.debug('started with %s', key) self.copy(key, key, metadata) def close(self): log.debug('started') self.conn_api.disconnect() self.conn_download.disconnect()
def test_dns_two(monkeypatch): monkeypatch.setattr(dugong, 'DNS_TEST_HOSTNAMES', (('grumpf.invalid', 80),)) with pytest.raises(dugong.DNSUnavailable): conn = HTTPConnection('foobar.invalid') conn.connect()
raise SystemExit('Can only do http') path_list.append(urlunsplit(('', '') + o[2:4] + ('',))) # Code from here on is included in documentation # start-example import asyncio import atexit from dugong import HTTPConnection, AioFuture # Get a MainLoop instance from the asyncio module to switch # between coroutines (and clean up at program exit) loop = asyncio.get_event_loop() atexit.register(loop.close) with HTTPConnection(hostname, port) as conn: # This generator function returns a coroutine that sends # all the requests. def send_requests(): for path in path_list: yield from conn.co_send_request('GET', path) # This generator function returns a coroutine that reads # all the responses def read_responses(): bodies = [] for path in path_list: resp = yield from conn.co_read_response() assert resp.status == 200 buf = yield from conn.co_readall() bodies.append(buf)
def _detect_features(self, hostname, port, ssl_context): '''Try to figure out the Swift version and supported features by examining the /info endpoint of the storage server. See https://docs.openstack.org/swift/latest/middleware.html#discoverability ''' if 'no-feature-detection' in self.options: log.debug('Skip feature detection') return if not port: port = 443 if ssl_context else 80 detected_features = Features() with HTTPConnection(hostname, port, proxy=self.proxy, ssl_context=ssl_context) as conn: conn.timeout = int(self.options.get('tcp-timeout', 20)) log.debug('GET /info') conn.send_request('GET', '/info') resp = conn.read_response() # 200, 401, 403 and 404 are all OK since the /info endpoint # may not be accessible (misconfiguration) or may not # exist (old Swift version). if resp.status not in (200, 401, 403, 404): log.error("Wrong server response.\n%s", self._dump_response(resp, body=conn.read(2048))) raise HTTPError(resp.status, resp.reason, resp.headers) if resp.status == 200: hit = re.match(r'^application/json(;\s*charset="?(.+?)"?)?$', resp.headers['content-type']) if not hit: log.error("Wrong server response. Expected json. Got: \n%s", self._dump_response(resp, body=conn.read(2048))) raise RuntimeError('Unexpected server reply') info = json.loads(conn.readall().decode(hit.group(2) or 'utf-8')) swift_info = info.get('swift', {}) log.debug('%s:%s/info returns %s', hostname, port, info) swift_version_string = swift_info.get('version', None) if swift_version_string and \ LooseVersion(swift_version_string) >= LooseVersion('2.8'): detected_features.has_copy = True # Default metadata value length constrain is 256 bytes # but the provider could configure another value. # We only decrease the chunk size since 255 is a big enough chunk size. max_meta_len = swift_info.get('max_meta_value_length', None) if isinstance(max_meta_len, int) and max_meta_len < 256: detected_features.max_meta_len = max_meta_len if info.get('bulk_delete', False): detected_features.has_bulk_delete = True bulk_delete = info['bulk_delete'] assert bulk_delete.get('max_failed_deletes', 1000) <= \ bulk_delete.get('max_deletes_per_request', 10000) assert bulk_delete.get('max_failed_deletes', 1000) > 0 # The block cache removal queue has a capacity of 1000. # We do not need bigger values than that. # We use max_failed_deletes instead of max_deletes_per_request # because then we can be sure even when all our delete requests # get rejected we get a complete error list back from the server. # If we would set the value higher, _delete_multi() would maybe # delete some entries from the *keys* list that did not get # deleted and would miss them in a retry. detected_features.max_deletes = min(1000, int(bulk_delete.get('max_failed_deletes', 1000))) log.info('Detected Swift features for %s:%s: %s', hostname, port, detected_features, extra=LOG_ONCE) else: log.debug('%s:%s/info not found or not accessible. Skip feature detection.', hostname, port) self.features = detected_features
def _get_conn(self): '''Obtain connection to server and authentication token''' log.debug('started') if 'no-ssl' in self.options: ssl_context = None else: ssl_context = self.ssl_context headers = CaseInsensitiveDict() headers['Content-Type'] = 'application/json' headers['Accept'] = 'application/json; charset="utf-8"' if ':' in self.login: (tenant, user) = self.login.split(':') else: tenant = None user = self.login auth_body = { 'auth': { 'passwordCredentials': { 'username': user, 'password': self.password } } } if tenant: auth_body['auth']['tenantName'] = tenant with HTTPConnection(self.hostname, port=self.port, proxy=self.proxy, ssl_context=ssl_context) as conn: conn.timeout = int(self.options.get('tcp-timeout', 20)) conn.send_request('POST', '/v2.0/tokens', headers=headers, body=json.dumps(auth_body).encode('utf-8')) resp = conn.read_response() if resp.status == 401: raise AuthorizationError(resp.reason) elif resp.status > 299 or resp.status < 200: raise HTTPError(resp.status, resp.reason, resp.headers) cat = json.loads(conn.read().decode('utf-8')) self.auth_token = cat['access']['token']['id'] avail_regions = [] for service in cat['access']['serviceCatalog']: if service['type'] != 'object-store': continue for endpoint in service['endpoints']: if endpoint['region'] != self.region: avail_regions.append(endpoint['region']) continue o = urlsplit(endpoint['publicURL']) self.auth_prefix = urllib.parse.unquote(o.path) if o.scheme == 'https': ssl_context = self.ssl_context elif o.scheme == 'http': ssl_context = None else: # fall through to scheme used for authentication pass self._detect_features(o.hostname, o.port, ssl_context) conn = HTTPConnection(o.hostname, o.port, proxy=self.proxy, ssl_context=ssl_context) conn.timeout = int(self.options.get('tcp-timeout', 20)) return conn if len(avail_regions) < 10: raise DanglingStorageURLError( self.container_name, 'No accessible object storage service found in region %s' ' (available regions: %s)' % (self.region, ', '.join(avail_regions))) else: raise DanglingStorageURLError( self.container_name, 'No accessible object storage service found in region %s' % self.region)
def _get_conn(self): '''Obtain connection to server and authentication token''' log.debug('started') if 'no-ssl' in self.options: ssl_context = None else: ssl_context = self.ssl_context headers = CaseInsensitiveDict() headers['Content-Type'] = 'application/json' headers['Accept'] = 'application/json; charset="utf-8"' if ':' in self.login: (tenant,user) = self.login.split(':') else: tenant = None user = self.login auth_body = { 'auth': { 'passwordCredentials': { 'username': user, 'password': self.password } }} if tenant: auth_body['auth']['tenantName'] = tenant with HTTPConnection(self.hostname, port=self.port, proxy=self.proxy, ssl_context=ssl_context) as conn: conn.timeout = int(self.options.get('tcp-timeout', 20)) conn.send_request('POST', '/v2.0/tokens', headers=headers, body=json.dumps(auth_body).encode('utf-8')) resp = conn.read_response() if resp.status == 401: raise AuthorizationError(resp.reason) elif resp.status > 299 or resp.status < 200: raise HTTPError(resp.status, resp.reason, resp.headers) cat = json.loads(conn.read().decode('utf-8')) self.auth_token = cat['access']['token']['id'] avail_regions = [] for service in cat['access']['serviceCatalog']: if service['type'] != 'object-store': continue for endpoint in service['endpoints']: if endpoint['region'] != self.region: avail_regions.append(endpoint['region']) continue o = urlsplit(endpoint['publicURL']) self.auth_prefix = urllib.parse.unquote(o.path) if o.scheme == 'https': ssl_context = self.ssl_context elif o.scheme == 'http': ssl_context = None else: # fall through to scheme used for authentication pass conn = HTTPConnection(o.hostname, o.port, proxy=self.proxy, ssl_context=ssl_context) conn.timeout = int(self.options.get('tcp-timeout', 20)) return conn if len(avail_regions) < 10: raise DanglingStorageURLError(self.container_name, 'No accessible object storage service found in region %s' ' (available regions: %s)' % (self.region, ', '.join(avail_regions))) else: raise DanglingStorageURLError(self.container_name, 'No accessible object storage service found in region %s' % self.region)
def test_dns_one(monkeypatch): monkeypatch.setattr(dugong, 'DNS_TEST_HOSTNAMES', (('localhost', 80), )) with pytest.raises(dugong.HostnameNotResolvable): conn = HTTPConnection('foobar.invalid') conn.connect()
def test_dns_two(monkeypatch): monkeypatch.setattr(dugong, 'DNS_TEST_HOSTNAMES', (('grumpf.invalid', 80), )) with pytest.raises(dugong.DNSUnavailable): conn = HTTPConnection('foobar.invalid') conn.connect()
def _get_conn(self): '''Obtain connection to server and authentication token''' log.debug('started') if 'no-ssl' in self.options: ssl_context = None else: ssl_context = self.ssl_context headers = CaseInsensitiveDict() headers['Content-Type'] = 'application/json' headers['Accept'] = 'application/json; charset="utf-8"' if ':' in self.login: (tenant,user) = self.login.split(':') else: tenant = None user = self.login domain = self.options.get('domain', None) if domain: if not tenant: raise ValueError("Tenant is required when Keystone v3 is used") # In simple cases where there's only one domain, the project domain # will be the same as the authentication domain, but this option # allows for them to be different project_domain = self.options.get('project-domain', domain) auth_body = { 'auth': { 'identity': { 'methods': ['password'], 'password': { 'user': { 'name': user, 'domain': { 'id': domain }, 'password': self.password } } }, 'scope': { 'project': { 'id': tenant, 'domain': { 'id': project_domain } } } } } auth_url_path = '/v3/auth/tokens' else: # If a domain is not specified, assume v2 auth_body = { 'auth': { 'passwordCredentials': { 'username': user, 'password': self.password } }} auth_url_path = '/v2.0/tokens' if tenant: auth_body['auth']['tenantName'] = tenant with HTTPConnection(self.hostname, port=self.port, proxy=self.proxy, ssl_context=ssl_context) as conn: conn.timeout = int(self.options.get('tcp-timeout', 20)) conn.send_request('POST', auth_url_path, headers=headers, body=json.dumps(auth_body).encode('utf-8')) resp = conn.read_response() if resp.status == 401: raise AuthorizationError(resp.reason) elif resp.status > 299 or resp.status < 200: raise HTTPError(resp.status, resp.reason, resp.headers) cat = json.loads(conn.read().decode('utf-8')) if self.options.get('domain', None): self.auth_token = resp.headers['X-Subject-Token'] service_catalog = cat['token']['catalog'] else: self.auth_token = cat['access']['token']['id'] service_catalog = cat['access']['serviceCatalog'] avail_regions = [] for service in service_catalog: if service['type'] != 'object-store': continue for endpoint in service['endpoints']: if endpoint['region'] != self.region: avail_regions.append(endpoint['region']) continue if 'publicURL' in endpoint: # The publicURL nomenclature is found in v2 catalogs o = urlsplit(endpoint['publicURL']) else: # Whereas v3 catalogs do 'interface' == 'public' and # 'url' for the URL itself if endpoint['interface'] != 'public': continue o = urlsplit(endpoint['url']) self.auth_prefix = urllib.parse.unquote(o.path) if o.scheme == 'https': ssl_context = self.ssl_context elif o.scheme == 'http': ssl_context = None else: # fall through to scheme used for authentication pass self._detect_features(o.hostname, o.port, ssl_context) conn = HTTPConnection(o.hostname, o.port, proxy=self.proxy, ssl_context=ssl_context) conn.timeout = int(self.options.get('tcp-timeout', 20)) return conn if len(avail_regions) < 10: raise DanglingStorageURLError(self.container_name, 'No accessible object storage service found in region %s' ' (available regions: %s)' % (self.region, ', '.join(avail_regions))) else: raise DanglingStorageURLError(self.container_name, 'No accessible object storage service found in region %s' % self.region)
# When running from HG repo, enable all warnings if os.path.exists(os.path.join(basedir, '.hg')): import warnings warnings.simplefilter('error') from dugong import HTTPConnection, BUFFER_SIZE for arg in sys.argv[1:]: url = urlsplit(arg) assert url.scheme == 'http' path = url.path if url.query: path += '?' + url.query with HTTPConnection(url.hostname, url.port) as conn: conn.send_request('GET', path) resp = conn.read_response() if resp.status != 200: raise SystemExit('%d %s' % (resp.status, resp.reason)) # Determine if we're reading text or binary data, and (in case of text), # what character set is being used. if 'Content-Type' not in resp.headers: type_ = 'application/octet-stream' else: type_ = resp.headers['Content-Type'] hit = re.match(r'(.+?)(?:; charset=(.+))?$', type_) if not hit: raise SystemExit('Unable to parse content-type: %s' % type_)