def _login(self): """ Login with backend and make a new connection to API and download server """ id_and_key = self.account_id + ':' + self.account_key basic_auth_string = 'Basic ' + str( base64.b64encode(bytes(id_and_key, 'UTF-8')), encoding='UTF-8') with HTTPConnection(self.authorize_hostname, 443, ssl_context=self.ssl_context) as conn: resp, body = self._do_request('GET', self.authorize_url, conn, auth_token=basic_auth_string) j = json.loads(str(body, encoding='UTF-8')) api_url = urllib.parse.urlparse(j['apiUrl']) download_url = urllib.parse.urlparse(j['downloadUrl']) self.api_host = api_url.hostname self.auth_token = j['authorizationToken'] self.download_host = download_url.hostname self.conn_api = HTTPConnection(self.api_host, 443, ssl_context=self.ssl_context) self.conn_download = HTTPConnection(self.download_host, 443, ssl_context=self.ssl_context)
def _get_conn(self): '''Obtain connection to server and authentication token''' log.debug('started') if 'no-ssl' in self.options: ssl_context = None else: ssl_context = self.ssl_context headers = CaseInsensitiveDict() headers['X-Auth-User'] = self.login headers['X-Auth-Key'] = self.password with HTTPConnection(self.hostname, self.port, proxy=self.proxy, ssl_context=ssl_context) as conn: conn.timeout = int(self.options.get('tcp-timeout', 20)) for auth_path in ('/v1.0', '/auth/v1.0'): log.debug('GET %s', auth_path) conn.send_request('GET', auth_path, headers=headers) resp = conn.read_response() if resp.status in (404, 412): log.debug('auth to %s failed, trying next path', auth_path) conn.discard() continue elif resp.status == 401: raise AuthorizationError(resp.reason) elif resp.status > 299 or resp.status < 200: raise HTTPError(resp.status, resp.reason, resp.headers) # Pylint can't infer SplitResult Types #pylint: disable=E1103 self.auth_token = resp.headers['X-Auth-Token'] o = urlsplit(resp.headers['X-Storage-Url']) self.auth_prefix = urllib.parse.unquote(o.path) if o.scheme == 'https': ssl_context = self.ssl_context elif o.scheme == 'http': ssl_context = None else: # fall through to scheme used for authentication pass conn = HTTPConnection(o.hostname, o.port, proxy=self.proxy, ssl_context=ssl_context) conn.timeout = int(self.options.get('tcp-timeout', 20)) return conn raise RuntimeError('No valid authentication path found')
def _get_access_token(self): log.info('Requesting new access token') headers = CaseInsensitiveDict() headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=utf-8' body = urllib.parse.urlencode({ 'client_id': OAUTH_CLIENT_ID, 'client_secret': OAUTH_CLIENT_SECRET, 'refresh_token': self.password, 'grant_type': 'refresh_token' }) conn = HTTPConnection('accounts.google.com', 443, proxy=self.proxy, ssl_context=self.ssl_context) try: conn.send_request('POST', '/o/oauth2/token', headers=headers, body=body.encode('utf-8')) resp = conn.read_response() json_resp = self._parse_json_response(resp, conn) if resp.status > 299 or resp.status < 200: assert 'error' in json_resp if 'error' in json_resp: raise AuthenticationError(json_resp['error']) else: self.access_token[self.password] = json_resp['access_token'] finally: conn.disconnect()
def test_http_proxy(http_server, monkeypatch, test_port): test_host = 'www.foobarz.invalid' test_path = '/someurl?barf' get_path = None def do_GET(self): nonlocal get_path get_path = self.path self.send_response(200) self.send_header("Content-Type", 'application/octet-stream') self.send_header("Content-Length", '0') self.end_headers() monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET) conn = HTTPConnection(test_host, test_port, proxy=(http_server.host, http_server.port)) try: conn.send_request('GET', test_path) resp = conn.read_response() assert resp.status == 200 conn.discard() finally: conn.disconnect() if test_port is None: exp_path = 'http://%s%s' % (test_host, test_path) else: exp_path = 'http://%s:%d%s' % (test_host, test_port, test_path) assert get_path == exp_path
def _get_conn(self): '''Return connection to server''' conn = HTTPConnection(self.hostname, self.port, proxy=self.proxy, ssl_context=self.ssl_context) conn.timeout = int(self.options.get('tcp-timeout', 20)) return conn
def close(self): """Close object and upload data""" log.debug('started with %s', self.key) if self.closed: # still call fh.close, may have generated an error before self.fh.close() return self.fh.seek(0) upload_auth_token, upload_url = self.backend._get_upload_url() upload_url = urllib.parse.urlparse(upload_url) with HTTPConnection(upload_url.hostname, 443, ssl_context=self.backend.ssl_context) as conn_up: headers = CaseInsensitiveDict() headers['X-Bz-File-Name'] = self.backend.prefix + self.key headers['Content-Type'] = 'application/octet-stream' headers['Content-Length'] = self.obj_size headers['X-Bz-Content-Sha1'] = self.sha1.hexdigest() if self.meta is None: self.meta = dict() self.backend._add_meta_headers(headers, self.meta) self.backend._do_request('POST', upload_url.path + '?' + upload_url.query, conn_up, headers=headers, body=self.fh, auth_token=upload_auth_token, body_size=self.obj_size) self.fh.close() self.closed = True
def _get_download_connection(self): if self.download_url is None: self._authorize_account() if self.download_connection is None: self.download_connection = HTTPConnection( self.download_url.hostname, 443, ssl_context=self.ssl_context) self.download_connection.timeout = self.tcp_timeout return self.download_connection
def _get_access_token(self): log.info('Requesting new access token') headers = CaseInsensitiveDict() headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=utf-8' body = urlencode({'client_id': OAUTH_CLIENT_ID, 'client_secret': OAUTH_CLIENT_SECRET, 'refresh_token': self.password, 'grant_type': 'refresh_token' }) conn = HTTPConnection('accounts.google.com', 443, proxy=self.proxy, ssl_context=self.ssl_context) try: conn.send_request('POST', '/o/oauth2/token', headers=headers, body=body.encode('utf-8')) resp = conn.read_response() if resp.status > 299 or resp.status < 200: raise HTTPError(resp.status, resp.reason, resp.headers) content_type = resp.headers.get('Content-Type', None) if content_type: hit = re.match(r'application/json(?:; charset="(.+)")?$', resp.headers['Content-Type'], re.IGNORECASE) else: hit = None if not hit: log.error('Unexpected server reply when refreshing access token:\n%s', self._dump_response(resp)) raise RuntimeError('Unable to parse server response') charset = hit.group(1) or 'utf-8' body = conn.readall().decode(charset) resp_json = json.loads(body) if not isinstance(resp_json, dict): log.error('Invalid json server response. Expected dict, got:\n%s', body) raise RuntimeError('Unable to parse server response') if 'error' in resp_json: raise AuthenticationError(resp_json['error']) if 'access_token' not in resp_json: log.error('Unable to find access token in server response:\n%s', body) raise RuntimeError('Unable to parse server response') self.access_token[self.password] = resp_json['access_token'] finally: conn.disconnect()
def test_invalid_ssl(): check_http_connection() # Don't load certificates context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context.options |= ssl.OP_NO_SSLv2 context.verify_mode = ssl.CERT_REQUIRED conn = HTTPConnection(SSL_TEST_HOST, ssl_context=context) with pytest.raises(ssl.SSLError): conn.send_request('GET', '/') conn.disconnect()
def test_connect_proxy(http_server, monkeypatch, test_port): test_host = 'www.foobarz.invalid' test_path = '/someurl?barf' connect_path = None def do_CONNECT(self): # Pretend we're the remote server too nonlocal connect_path connect_path = self.path self.send_response(200) self.end_headers() self.close_connection = 0 monkeypatch.setattr(MockRequestHandler, 'do_CONNECT', do_CONNECT, raising=False) get_path = None def do_GET(self): nonlocal get_path get_path = self.path self.send_response(200) self.send_header("Content-Type", 'application/octet-stream') self.send_header("Content-Length", '0') self.end_headers() monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET) # We don't *actually* want to establish SSL, that'd be # to complex for our mock server monkeypatch.setattr('ssl.match_hostname', lambda x, y: True) conn = HTTPConnection(test_host, test_port, proxy=(http_server.host, http_server.port), ssl_context=FakeSSLContext()) try: conn.send_request('GET', test_path) resp = conn.read_response() assert resp.status == 200 conn.discard() finally: conn.disconnect() if test_port is None: test_port = 443 exp_path = '%s:%d' % (test_host, test_port) assert connect_path == exp_path assert get_path == test_path
def main(): if len(sys.argv) != 2: raise SystemExit('Usage: %s <url>' % sys.argv[0]) url = sys.argv[1] url_els = urlsplit(url) if url_els.scheme == 'https': ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) ssl_context.options |= ssl.OP_NO_SSLv2 ssl_context.verify_mode = ssl.CERT_REQUIRED ssl_context.set_default_verify_paths() else: ssl_context = None with HTTPConnection(url_els.hostname, port=url_els.port, ssl_context=ssl_context) as conn: path = urlunsplit(('', '') + url_els[2:4] + ('', )) or '/' conn.send_request('GET', path) resp = conn.read_response() if resp.status != 200: raise SystemExit('%d %s' % (resp.status, resp.reason)) # Determine if we're reading text or binary data, and (in case of text), # what character set is being used. if 'Content-Type' not in resp.headers: type_ = 'application/octet-stream' else: type_ = resp.headers['Content-Type'] hit = re.match(r'text/x?html(?:; charset=(.+))?$', type_) if not hit: raise SystemExit('Server did not send html but %s' % type_) if hit.group(1): charset = hit.group(1) else: charset = 'latin1' html_stream = TextIOWrapper(conn, encoding=charset) parser = LinkExtractor() while True: buf = html_stream.read(16 * 1024) if not buf: break parser.feed(buf) for link in parser.links: print(urljoin(url, link))
def conn(request, http_server): if http_server.use_ssl: ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) ssl_context.options |= ssl.OP_NO_SSLv2 ssl_context.verify_mode = ssl.CERT_REQUIRED ssl_context.load_verify_locations( cafile=os.path.join(TEST_DIR, 'ca.crt')) else: ssl_context = None conn = HTTPConnection(http_server.host, port=http_server.port, ssl_context=ssl_context) request.addfinalizer(conn.disconnect) return conn
def test_connect_ssl(): check_http_connection() ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) ssl_context.options |= ssl.OP_NO_SSLv2 ssl_context.verify_mode = ssl.CERT_REQUIRED ssl_context.set_default_verify_paths() conn = HTTPConnection(SSL_TEST_HOST, ssl_context=ssl_context) conn.send_request('GET', '/') resp = conn.read_response() assert resp.status in (200, 301, 302) assert resp.path == '/' conn.discard() conn.disconnect()
def __call__(self, url: str, method: str = 'GET', body=None, headers=None, timeout=None): # https://github.com/googleapis/google-auth-library-python/issues/318 if not isinstance(body, bytes): body = str(body).encode('ascii') if timeout is not None: raise ValueError('*timeout* argument is not supported') hit = re.match(r'^(https?)://([^:/]+)(?::(\d+))?(.*)$', url) if not hit: raise ValueError('Unsupported URL: ' + url) if hit.group(1) == 'https': ssl_context = self.ssl_context proxy = self.ssl_proxy else: ssl_context = None proxy = self.proxy hostname = hit.group(2) if hit.group(3): port = int(hit.group(3)) elif ssl_context: port = 443 else: port = 80 path = hit.group(4) try: conn = self.conn_pool[(hostname, port)] except KeyError: conn = HTTPConnection(hostname, port, proxy=proxy, ssl_context=ssl_context) self.conn_pool[(hostname, port)] = conn try: conn.send_request(method, path, headers=headers, body=body) resp = conn.read_response() except (dugong.ConnectionClosed, dugong.InvalidResponse, dugong.UnsupportedResponse, dugong.ConnectionTimedOut, dugong.HostnameNotResolvable, dugong.DNSUnavailable, ssl.SSLError) as exc: raise g_auth.exceptions.TransportError(exc) return Namespace(status=resp.status, headers=resp.headers, data=conn.readall())
def _authorize_account(self): '''Authorize API calls''' authorize_host = 'api.backblazeb2.com' authorize_url = api_url_prefix + 'b2_authorize_account' id_and_key = self.b2_application_key_id + ':' + self.b2_application_key basic_auth_string = 'Basic ' + str( base64.b64encode(bytes(id_and_key, 'UTF-8')), encoding='UTF-8') with HTTPConnection(authorize_host, 443, ssl_context=self.ssl_context) as connection: headers = CaseInsensitiveDict() headers['Authorization'] = basic_auth_string connection.send_request('GET', authorize_url, headers=headers, body=None) response = connection.read_response() response_body = connection.readall() if response.status != 200: raise RuntimeError('Authorization failed.') j = json.loads(response_body.decode('utf-8')) self.account_id = j['accountId'] allowed_info = j.get('allowed') if allowed_info.get('bucketId'): self.bucket_id = allowed_info.get('bucketId') if allowed_info.get('bucketName') != self.bucket_name: raise RuntimeError( 'Provided API key can not access desired bucket.') if not self._check_key_capabilities(allowed_info): raise RuntimeError( 'Provided API key does not have the required capabilities.' ) self.api_url = urlparse(j['apiUrl']) self.download_url = urlparse(j['downloadUrl']) self.authorization_token = j['authorizationToken']
def _request_upload_url_info(self): request_data = {'bucketId': self._get_bucket_id()} response = self._do_api_call('b2_get_upload_url', request_data) new_upload_url = urlparse(response['uploadUrl']) new_authorization_token = response['authorizationToken'] upload_connection = HTTPConnection(new_upload_url.hostname, 443, ssl_context=self.ssl_context) upload_connection.timeout = self.tcp_timeout return { 'hostname': new_upload_url.hostname, 'connection': upload_connection, 'path': new_upload_url.path, 'authorizationToken': new_authorization_token, 'isUploading': False }
raise SystemExit('Can only do http') path_list.append(urlunsplit(('', '') + o[2:4] + ('',))) # Code from here on is included in documentation # start-example import asyncio import atexit from dugong import HTTPConnection, AioFuture # Get a MainLoop instance from the asyncio module to switch # between coroutines (and clean up at program exit) loop = asyncio.get_event_loop() atexit.register(loop.close) with HTTPConnection(hostname, port) as conn: # This generator function returns a coroutine that sends # all the requests. def send_requests(): for path in path_list: yield from conn.co_send_request('GET', path) # This generator function returns a coroutine that reads # all the responses def read_responses(): bodies = [] for path in path_list: resp = yield from conn.co_read_response() assert resp.status == 200 buf = yield from conn.co_readall() bodies.append(buf)
def _detect_features(self, hostname, port, ssl_context): '''Try to figure out the Swift version and supported features by examining the /info endpoint of the storage server. See https://docs.openstack.org/swift/latest/middleware.html#discoverability ''' if 'no-feature-detection' in self.options: log.debug('Skip feature detection') return if not port: port = 443 if ssl_context else 80 detected_features = Features() with HTTPConnection(hostname, port, proxy=self.proxy, ssl_context=ssl_context) as conn: conn.timeout = int(self.options.get('tcp-timeout', 20)) log.debug('GET /info') conn.send_request('GET', '/info') resp = conn.read_response() # 200, 401, 403 and 404 are all OK since the /info endpoint # may not be accessible (misconfiguration) or may not # exist (old Swift version). if resp.status not in (200, 401, 403, 404): log.error("Wrong server response.\n%s", self._dump_response(resp, body=conn.read(2048))) raise HTTPError(resp.status, resp.reason, resp.headers) if resp.status == 200: hit = re.match(r'^application/json(;\s*charset="?(.+?)"?)?$', resp.headers['content-type']) if not hit: log.error("Wrong server response. Expected json. Got: \n%s", self._dump_response(resp, body=conn.read(2048))) raise RuntimeError('Unexpected server reply') info = json.loads(conn.readall().decode(hit.group(2) or 'utf-8')) swift_info = info.get('swift', {}) log.debug('%s:%s/info returns %s', hostname, port, info) swift_version_string = swift_info.get('version', None) if swift_version_string and \ LooseVersion(swift_version_string) >= LooseVersion('2.8'): detected_features.has_copy = True # Default metadata value length constrain is 256 bytes # but the provider could configure another value. # We only decrease the chunk size since 255 is a big enough chunk size. max_meta_len = swift_info.get('max_meta_value_length', None) if isinstance(max_meta_len, int) and max_meta_len < 256: detected_features.max_meta_len = max_meta_len if info.get('bulk_delete', False): detected_features.has_bulk_delete = True bulk_delete = info['bulk_delete'] assert bulk_delete.get('max_failed_deletes', 1000) <= \ bulk_delete.get('max_deletes_per_request', 10000) assert bulk_delete.get('max_failed_deletes', 1000) > 0 # The block cache removal queue has a capacity of 1000. # We do not need bigger values than that. # We use max_failed_deletes instead of max_deletes_per_request # because then we can be sure even when all our delete requests # get rejected we get a complete error list back from the server. # If we would set the value higher, _delete_multi() would maybe # delete some entries from the *keys* list that did not get # deleted and would miss them in a retry. detected_features.max_deletes = min(1000, int(bulk_delete.get('max_failed_deletes', 1000))) log.info('Detected Swift features for %s:%s: %s', hostname, port, detected_features, extra=LOG_ONCE) else: log.debug('%s:%s/info not found or not accessible. Skip feature detection.', hostname, port) self.features = detected_features
def _get_conn(self): '''Obtain connection to server and authentication token''' log.debug('started') if 'no-ssl' in self.options: ssl_context = None else: ssl_context = self.ssl_context headers = CaseInsensitiveDict() headers['Content-Type'] = 'application/json' headers['Accept'] = 'application/json; charset="utf-8"' if ':' in self.login: (tenant, user) = self.login.split(':') else: tenant = None user = self.login auth_body = { 'auth': { 'passwordCredentials': { 'username': user, 'password': self.password } } } if tenant: auth_body['auth']['tenantName'] = tenant with HTTPConnection(self.hostname, port=self.port, proxy=self.proxy, ssl_context=ssl_context) as conn: conn.timeout = int(self.options.get('tcp-timeout', 20)) conn.send_request('POST', '/v2.0/tokens', headers=headers, body=json.dumps(auth_body).encode('utf-8')) resp = conn.read_response() if resp.status == 401: raise AuthorizationError(resp.reason) elif resp.status > 299 or resp.status < 200: raise HTTPError(resp.status, resp.reason, resp.headers) cat = json.loads(conn.read().decode('utf-8')) self.auth_token = cat['access']['token']['id'] avail_regions = [] for service in cat['access']['serviceCatalog']: if service['type'] != 'object-store': continue for endpoint in service['endpoints']: if endpoint['region'] != self.region: avail_regions.append(endpoint['region']) continue o = urlsplit(endpoint['publicURL']) self.auth_prefix = urllib.parse.unquote(o.path) if o.scheme == 'https': ssl_context = self.ssl_context elif o.scheme == 'http': ssl_context = None else: # fall through to scheme used for authentication pass self._detect_features(o.hostname, o.port, ssl_context) conn = HTTPConnection(o.hostname, o.port, proxy=self.proxy, ssl_context=ssl_context) conn.timeout = int(self.options.get('tcp-timeout', 20)) return conn if len(avail_regions) < 10: raise DanglingStorageURLError( self.container_name, 'No accessible object storage service found in region %s' ' (available regions: %s)' % (self.region, ', '.join(avail_regions))) else: raise DanglingStorageURLError( self.container_name, 'No accessible object storage service found in region %s' % self.region)
def test_dns_two(monkeypatch): monkeypatch.setattr(dugong, 'DNS_TEST_HOSTNAMES', (('grumpf.invalid', 80), )) with pytest.raises(dugong.DNSUnavailable): conn = HTTPConnection('foobar.invalid') conn.connect()
def test_dns_one(monkeypatch): monkeypatch.setattr(dugong, 'DNS_TEST_HOSTNAMES', (('localhost', 80), )) with pytest.raises(dugong.HostnameNotResolvable): conn = HTTPConnection('foobar.invalid') conn.connect()
def _get_conn(self): '''Obtain connection to server and authentication token''' log.debug('started') if 'no-ssl' in self.options: ssl_context = None else: ssl_context = self.ssl_context headers = CaseInsensitiveDict() headers['Content-Type'] = 'application/json' headers['Accept'] = 'application/json; charset="utf-8"' if ':' in self.login: (tenant,user) = self.login.split(':') else: tenant = None user = self.login domain = self.options.get('domain', None) if domain: if not tenant: raise ValueError("Tenant is required when Keystone v3 is used") # In simple cases where there's only one domain, the project domain # will be the same as the authentication domain, but this option # allows for them to be different project_domain = self.options.get('project-domain', domain) auth_body = { 'auth': { 'identity': { 'methods': ['password'], 'password': { 'user': { 'name': user, 'domain': { 'id': domain }, 'password': self.password } } }, 'scope': { 'project': { 'id': tenant, 'domain': { 'id': project_domain } } } } } auth_url_path = '/v3/auth/tokens' else: # If a domain is not specified, assume v2 auth_body = { 'auth': { 'passwordCredentials': { 'username': user, 'password': self.password } }} auth_url_path = '/v2.0/tokens' if tenant: auth_body['auth']['tenantName'] = tenant with HTTPConnection(self.hostname, port=self.port, proxy=self.proxy, ssl_context=ssl_context) as conn: conn.timeout = int(self.options.get('tcp-timeout', 20)) conn.send_request('POST', auth_url_path, headers=headers, body=json.dumps(auth_body).encode('utf-8')) resp = conn.read_response() if resp.status == 401: raise AuthorizationError(resp.reason) elif resp.status > 299 or resp.status < 200: raise HTTPError(resp.status, resp.reason, resp.headers) cat = json.loads(conn.read().decode('utf-8')) if self.options.get('domain', None): self.auth_token = resp.headers['X-Subject-Token'] service_catalog = cat['token']['catalog'] else: self.auth_token = cat['access']['token']['id'] service_catalog = cat['access']['serviceCatalog'] avail_regions = [] for service in service_catalog: if service['type'] != 'object-store': continue for endpoint in service['endpoints']: if endpoint['region'] != self.region: avail_regions.append(endpoint['region']) continue if 'publicURL' in endpoint: # The publicURL nomenclature is found in v2 catalogs o = urlsplit(endpoint['publicURL']) else: # Whereas v3 catalogs do 'interface' == 'public' and # 'url' for the URL itself if endpoint['interface'] != 'public': continue o = urlsplit(endpoint['url']) self.auth_prefix = urllib.parse.unquote(o.path) if o.scheme == 'https': ssl_context = self.ssl_context elif o.scheme == 'http': ssl_context = None else: # fall through to scheme used for authentication pass self._detect_features(o.hostname, o.port, ssl_context) conn = HTTPConnection(o.hostname, o.port, proxy=self.proxy, ssl_context=ssl_context) conn.timeout = int(self.options.get('tcp-timeout', 20)) return conn if len(avail_regions) < 10: raise DanglingStorageURLError(self.container_name, 'No accessible object storage service found in region %s' ' (available regions: %s)' % (self.region, ', '.join(avail_regions))) else: raise DanglingStorageURLError(self.container_name, 'No accessible object storage service found in region %s' % self.region)
# When running from HG repo, enable all warnings if os.path.exists(os.path.join(basedir, '.hg')): import warnings warnings.simplefilter('error') from dugong import HTTPConnection, BUFFER_SIZE for arg in sys.argv[1:]: url = urlsplit(arg) assert url.scheme == 'http' path = url.path if url.query: path += '?' + url.query with HTTPConnection(url.hostname, url.port) as conn: conn.send_request('GET', path) resp = conn.read_response() if resp.status != 200: raise SystemExit('%d %s' % (resp.status, resp.reason)) # Determine if we're reading text or binary data, and (in case of text), # what character set is being used. if 'Content-Type' not in resp.headers: type_ = 'application/octet-stream' else: type_ = resp.headers['Content-Type'] hit = re.match(r'(.+?)(?:; charset=(.+))?$', type_) if not hit: raise SystemExit('Unable to parse content-type: %s' % type_)