class TestAWSConnection(unittest.TestCase): @patch('patroni.scripts.aws.requests_get', Mock(return_value=urllib3.HTTPResponse( status=200, body=b'{"instanceId": "012345", "region": "eu-west-1"}'))) def setUp(self): self.conn = AWSConnection('test') def test_on_role_change(self): self.assertTrue(self.conn.on_role_change('master')) with patch.object(MockVolumes, 'filter', Mock(return_value=[])): self.conn._retry.max_tries = 1 self.assertFalse(self.conn.on_role_change('master')) @patch('patroni.scripts.aws.requests_get', Mock(side_effect=Exception('foo'))) def test_non_aws(self): conn = AWSConnection('test') self.assertFalse(conn.on_role_change("master")) @patch('patroni.scripts.aws.requests_get', Mock(return_value=urllib3.HTTPResponse(status=200, body=b'foo'))) def test_aws_bizare_response(self): conn = AWSConnection('test') self.assertFalse(conn.aws_available()) @patch('patroni.scripts.aws.requests_get', Mock(return_value=urllib3.HTTPResponse(status=503, body=b'Error'))) @patch('sys.exit', Mock()) def test_main(self): self.assertIsNone(_main()) sys.argv = ['aws.py', 'on_start', 'replica', 'foo'] self.assertIsNone(_main())
def request(self, *args, **kwargs): self._tc.assertTrue(len(self._reqs) > 0) r = self._reqs.pop(0) self._tc.maxDiff = None self._tc.assertEqual(r[0], args) self._tc.assertEqual(r[1], kwargs) return urllib3.HTTPResponse(status=200, body=b'test')
def testQueryAPI(self): """Tests that query_api returns expected body.""" self.mock_http.request.return_value = urllib3.HTTPResponse( body=b'{"expected": "body"}') actual_response = self.deputy_instance.query_api( 'resource/Roster/QUERY', deputy.Query()) self.assertEqual(actual_response, {'expected': 'body'})
def test_request_with_headers(self): # Given connection = self.closing_http() headers = {'Xtra Key': 'Xtra Value'} http_response = urllib3.HTTPResponse(headers=headers) request = self.patch('urllib3.PoolManager.request', return_value=http_response) retry = self.patch('urllib3.util.Retry') # When response, _ = connection.request(method=REQUEST_METHOD, url=REQUEST_URL, headers=headers) # Then request.assert_called_once_with(REQUEST_METHOD, REQUEST_URL, headers=dict(headers, connection='close'), retries=retry(raise_on_redirect=False, redirect=5)) self.assertEqual( { 'content-location': REQUEST_URL, 'status': str(http_response.status), 'xtra key': 'Xtra Value' }, response)
def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None): self.add_headers(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies) # Default returns 200 and the same body message status = 200 body = request.body or "No Body" # check headers are present if self.__checkHeaders(request.headers): # Check signature is valid timestamp = request.headers[self._protocol.TIMESTAMP_HEADER] signature = self._protocol.signRequest(request.url, request.method, params=None, body=request.body, timestamp=timestamp) signatureHeaders = self._protocol.getHeaders(signature) for name, value in signatureHeaders.items(): _v = request.headers.get(name) if not _v or _v != value: status = 403 body = json.dumps({"error" : {"code" : 403, "message": "Headers {} is not correct".format(name)}}) break else: status = 403 body = json.dumps({"error" : {"code" : 403, "message": "Headers are missing"}}) headers = urllib3.response.HTTPHeaderDict(request.headers) if hasattr(body, 'read'): body.seek(0) else: body = BytesIO(body.encode('utf8')) response = urllib3.HTTPResponse(body, headers, status, preload_content=False) return self.build_response(request, response)
def test_request(self): # Given connection = self.closing_http() http_response = urllib3.HTTPResponse() request = self.patch('urllib3.PoolManager.request', return_value=http_response) retry = self.patch('urllib3.util.Retry') # When response, data = connection.request(method=REQUEST_METHOD, url=REQUEST_URL) # Then request.assert_called_once_with(REQUEST_METHOD, REQUEST_URL, headers={'connection': 'close'}, retries=retry(raise_on_redirect=False, redirect=5)) self.assertEqual( { 'content-location': REQUEST_URL, 'status': str(http_response.status) }, response) self.assertEqual(http_response.status, response.status) self.assertEqual(http_response.reason, response.reason) self.assertEqual(http_response.version, response.version) self.assertEqual(http_response.data, data)
def getLearningPlanData(request): # Send the body of the data data = { "grant_type": "client_credentials", "scope": { "userId": "SFADMIN", "companyId": "talenteam", "userType": "user", "resourceType": "learning_public_api" } } headers = {'Authorization': '', 'Content-Type': 'application/json'} url = 'https://talenteam-stage.plateau.com/learning/oauth-api/rest/v1/token' response = requests.post(url, data=json.dumps(data), headers=headers) data = response.json() access_token = data['access_token'] if (access_token): http = urllib3.PoolManager() url = 'https://talenteam-stage.plateau.com/learning/odatav4/public/user/learningPlan/v1/UserTodoLearningItems?$filter=criteria/includeDeeplink eq true and criteria/includeLearnerActions eq true&$format=json&$select=title' token = f"Bearer {access_token}" print(token) r = http.request('GET', url, headers={ 'Authorization': token, 'Content-Type': 'application/json' }) print(r.data) return urllib3.HTTPResponse("Successful call")
def request(self, *actual_request_target, **actual_request_headers_and_body): self._tc.assertTrue(len(self._reqs) > 0) expected_results = self._reqs.pop(0) self._tc.maxDiff = None expected_request_target = expected_results[0] # The expected HTTP method and URL path. expected_request_headers_and_body = expected_results[1] # dict that contains the expected body, headers self._tc.assertEqual(expected_request_target, actual_request_target) # actual_request_headers_and_body is a dict that contains the actual body, headers for k, expected in expected_request_headers_and_body.items(): self._tc.assertIn(k, actual_request_headers_and_body) if k == 'body': actual_body = actual_request_headers_and_body[k] self._tc.assertEqual(expected, actual_body) elif k == 'headers': actual_headers = actual_request_headers_and_body[k] for expected_header_name, expected_header_value in expected.items(): # Validate the generated request contains the expected header. self._tc.assertIn(expected_header_name, actual_headers) actual_header_value = actual_headers[expected_header_name] # Compare the actual value of the header against the expected value. pattern = re.compile(expected_header_value) m = pattern.match(actual_header_value) self._tc.assertTrue(m, msg="Expected:\n{0}\nActual:\n{1}".format( expected_header_value,actual_header_value)) if expected_header_name == 'Authorization': self._validate_authorization_header( expected_request_target, actual_headers, actual_header_value) elif k == 'timeout': self._tc.assertEqual(expected, actual_request_headers_and_body[k]) return urllib3.HTTPResponse(status=200, body=b'test')
class TestExhibitor(unittest.TestCase): @patch('urllib3.PoolManager.request', Mock(return_value=urllib3.HTTPResponse( status=200, body=b'{"servers":["127.0.0.1","127.0.0.2","127.0.0.3"],"port":2181}'))) @patch('patroni.dcs.zookeeper.PatroniKazooClient', MockKazooClient) def setUp(self): self.e = Exhibitor({'hosts': ['localhost', 'exhibitor'], 'port': 8181, 'scope': 'test', 'name': 'foo', 'ttl': 30, 'retry_timeout': 10}) @patch.object(ExhibitorEnsembleProvider, 'poll', Mock(return_value=True)) def test_get_cluster(self): self.assertRaises(ZooKeeperError, self.e.get_cluster)
def testQueryAPI_PagesRequest(self, _, page_depth, result_length): """Tests that multiple requests are made when 500 records in response.""" long_response = [] for _ in range(0, 500): long_response.append({'dummy': 'data'}) self.mock_http.request.side_effect = [ urllib3.HTTPResponse( body=bytes(json.dumps(long_response), 'utf-8')), urllib3.HTTPResponse( body=bytes(json.dumps(long_response), 'utf-8')), urllib3.HTTPResponse( body=bytes(json.dumps({'dummy': 'data'}), 'utf-8')) ] deputy_instance = deputy.Deputy( 'https://example.au.deputy.com/api/v1/', self.mock_http, '12345', None, 10000, True, page_depth) result = deputy_instance.query_api('resource/Roster/QUERY', deputy.Query()) self.assertEqual(len(result), result_length)
def __response(cls, body: typing.Union[str, bytes], status: int = 200, content_type: str = json_content_type, headers: typing.Optional[dict[str, str]] = None, preload_content: bool = True) -> urllib3.HTTPResponse: if headers is None: headers = {} headers.update(cls.headers_for_content_type(content_type)) return urllib3.HTTPResponse(body, headers=headers, status=status, preload_content=preload_content)
def test_get_recent_history_of_user(self): with open(path.join(path.dirname(__file__), 'resource/recent_history.json'), 'r', encoding='utf-8') as f: try: str = f.read() except: f.close() response = bytes(str, encoding='utf-8') r = urllib3.HTTPResponse(body=response) urllib3.PoolManager.request = MagicMock(return_value=r) result = asyncio.run(csgo_5e_query.get_recent_history_of_user('test')) self.assertEqual(result[0]['rating'], "23.39")
def make_response(method, url, body=json.dumps({}), headers={}, status=200): from io import BytesIO req = requests.Request(method.upper(), url).prepare() body_stream = BytesIO(body) if isinstance( body, bytes) else BytesIO(body.encode()) headers = urllib3.response.HTTPHeaderDict(headers) resp = urllib3.HTTPResponse(body_stream, headers, status, preload_content=False) return requests.adapters.HTTPAdapter().build_response(req, resp)
def get_urlopen_mock(body=DEFAULT_BODY_CONTENT, headers=None, status=200): mockHttpResponse = Mock(name='httplib.HTTPResponse') headers = urllib3.response.HTTPHeaderDict(headers) if not hasattr(body, 'read'): body = BytesIO(body) else: body.seek(0) urllib3_response = urllib3.HTTPResponse(body, headers, status, preload_content=False, original_response=mockHttpResponse) return MagicMock(return_value=urllib3_response)
def test_request_with_fields(self): # Given connection = self.closing_http() http_response = urllib3.HTTPResponse() request = self.patch('urllib3.PoolManager.request', return_value=http_response) retry = self.patch('urllib3.util.Retry') fields = object() # When connection.request(method=REQUEST_METHOD, url=REQUEST_URL, fields=fields) # Then request.assert_called_once_with(REQUEST_METHOD, REQUEST_URL, fields=fields, headers=dict(connection='close'), retries=retry(raise_on_redirect=False, redirect=5))
def _submit_requests(self, method, url, body=None, headers=None, query_params=None, fields=None, preload_content=None, timeout=None, encode_multipart=None): if url.startswith('papi://PAPI_LOCAL_HOST'): # operating on-cluster. use isi.rest with papi.sock instead of urllib3 # so we can avoid re-authenticating. if 'Authorization' in headers: del headers['Authorization'] if fields is not None: # post parameters should be passed in the message body body = fields elif body is None: # both body and fields are none, pass in an empty body body = '{}' if query_params is not None: query_params = dict(query_params) # the body field must be a string if not isinstance(body, basestring): body = json.dumps(body) url_parts = url.partition('/platform/')[2].split('/') # isi.rest will re-encode the url so decode each part. this is stupid # but needed unless someone knows a way to get send_rest_request() to # not re-encode the url parts. for i in range(0, len(url_parts)): url_parts[i] = unquote(url_parts[i]) response = isi.rest.send_rest_request(isi.rest.PAPI_SOCKET_PATH, method=method, uri=url_parts, query_args=query_params, headers=headers, body=body, timeout=120) # we only use the headers, status, reason and body of a urllib3 # response object. the response from isi.rest.set_rest_request is: # [status, headers subset, body] reason = 'Response code: ' + str(response[0]) if 'status' in response[1]: reason = response[1]['status'] # the contents of the status field are: 'XXX reason_string' reason = reason[4:] r = urllib3.HTTPResponse(headers=response[1], status=response[0], reason=reason, body=response[2]) else: if method in ['POST', 'PUT', 'PATCH', 'OPTIONS', 'DELETE']: if query_params: url += '?' + urlencode(query_params) elif method in ['GET', 'HEAD']: fields = query_params if encode_multipart is not None: r = self.pool_manager.request( method, url, body=body, fields=fields, preload_content=preload_content, timeout=timeout, headers=headers, encode_multipart=encode_multipart) else: r = self.pool_manager.request(method, url, body=body, fields=fields, preload_content=preload_content, timeout=timeout, headers=headers) return r
async def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ) -> requests.Response: urlparts = urlparse(request.url) hostname = urlparts.hostname port = urlparts.port if port is None: port = {"http": 80, "https": 443}[urlparts.scheme] target = urlparts.path if urlparts.query: target += "?" + urlparts.query headers = [("host", urlparts.netloc)] + list(request.headers.items()) conn_kwargs = {"ssl": no_verify()} if urlparts.scheme == "https" else {} if isinstance(timeout, tuple): connect_timeout, read_timeout = timeout else: connect_timeout = timeout read_timeout = timeout try: reader, writer = await asyncio.wait_for( asyncio.open_connection(hostname, port, **conn_kwargs), connect_timeout ) except asyncio.TimeoutError: raise requests.ConnectTimeout() conn = h11.Connection(our_role=h11.CLIENT) message = h11.Request(method=request.method, target=target, headers=headers) data = conn.send(message) writer.write(data) if request.body: message = h11.Data(data=request.body.encode("utf-8")) data = conn.send(message) writer.write(data) message = h11.EndOfMessage() data = conn.send(message) writer.write(data) status_code = 0 headers = [] reason = b"" buffer = io.BytesIO() while True: event = conn.next_event() event_type = type(event) if event_type is h11.NEED_DATA: try: data = await asyncio.wait_for(reader.read(2048), read_timeout) except asyncio.TimeoutError: raise requests.ReadTimeout() conn.receive_data(data) elif event_type is h11.Response: status_code = event.status_code headers = [ (key.decode(), value.decode()) for key, value in event.headers ] reason = event.reason elif event_type is h11.Data: buffer.write(event.data) elif event_type is h11.EndOfMessage: buffer.seek(0) break writer.close() if hasattr(writer, "wait_closed"): await writer.wait_closed() resp = urllib3.HTTPResponse( body=buffer, headers=headers, status=status_code, reason=reason, preload_content=False, ) return self.build_response(request, resp)
async def send(self, request: requests.PreparedRequest, *args: typing.Any, **kwargs: typing.Any) -> requests.Response: urlparts = urlparse(request.url) hostname = urlparts.hostname port = urlparts.port if port is None: port = {"http": 80, "https": 443}[urlparts.scheme] target = urlparts.path if urlparts.query: target += "?" + urlparts.query headers = [("host", urlparts.netloc)] + list(request.headers.items()) conn_kwargs = { 'ssl': no_verify() } if urlparts.scheme == 'https' else {} reader, writer = await asyncio.open_connection(hostname, port, **conn_kwargs) conn = h11.Connection(our_role=h11.CLIENT) message = h11.Request(method=request.method, target=target, headers=headers) data = conn.send(message) writer.write(data) if request.body: message = h11.Data(data=request.body.encode("utf-8")) data = conn.send(message) writer.write(data) message = h11.EndOfMessage() data = conn.send(message) writer.write(data) status_code = 0 headers = [] reason = b"" buffer = io.BytesIO() while True: event = conn.next_event() event_type = type(event) if event_type is h11.NEED_DATA: data = await reader.read(2048) conn.receive_data(data) elif event_type is h11.Response: status_code = event.status_code headers = [(key.decode(), value.decode()) for key, value in event.headers] reason = event.reason elif event_type is h11.Data: buffer.write(event.data) elif event_type is h11.EndOfMessage: buffer.seek(0) break writer.close() if hasattr(writer, 'wait_closed'): await writer.wait_closed() resp = urllib3.HTTPResponse( body=buffer, headers=headers, status=status_code, reason=reason, preload_content=False, ) return self.build_response(request, resp)
def urllib3_http_request(self, url, values=None, headers={}, cacheTime=None, encoding=None, errors=None, timeout=0, immediate=False, sleep=0, data=None, opener=None, sandbox=None, follow_redirects=True, basic_auth=None, method=None): if cacheTime == None: cacheTime = self.cache_time pos = url.rfind('#') if pos > 0: url = url[:pos] if values and not data: data = urllib.urlencode(values) url_cache = None if self._http_caching_enabled: if cacheTime > 0: cache_mgr = self._cache_mgr if cache_mgr.item_count > self._core.config.http_cache_max_items + self._core.config.http_cache_max_items_grace: cache_mgr.trim(self._core.config.http_cache_max_size, self._core.config.http_cache_max_items) url_cache = cache_mgr[url] url_cache.set_expiry_interval(cacheTime) else: del self._cache_mgr[url] if url_cache != None and url_cache['content'] and not url_cache.expired: content_type = url_cache.headers.get('Content-Type', '') if self._core.plugin_class == 'Agent' and not _content_type_allowed( content_type): self._core.log.debug( "Removing cached data for '%s' (content type '%s' not cacheable in Agent plug-ins)", url, content_type) manager = url_cache._manager del manager[url] else: self._core.log.debug("Fetching '%s' from the HTTP cache", url) res = urllib3.HTTPResponse(url_cache['content'], headers=url_cache.headers) return res h = dict(self.default_headers) h.update({'connection': 'keep-alive'}) if sandbox: h.update(sandbox.custom_headers) h.update(headers) if 'PLEXTOKEN' in os.environ and len( os.environ['PLEXTOKEN']) > 0 and h is not None and url.find( 'http://127.0.0.1') == 0: h['X-Plex-Token'] = os.environ['PLEXTOKEN'] if basic_auth != None: h['Authorization'] = self.generate_basic_auth_header(*basic_auth) if url.startswith(config.kinopoisk.api.base[:-2]): h.update({ 'clientDate': datetime.now().strftime("%H:%M %d.%m.%Y"), 'x-timestamp': str(int(round(time.time() * 1000))) }) h.update({ 'x-signature': self._core.data.hashing.md5( url[len(config.kinopoisk.api.base[:-2]):] + h.get('x-timestamp') + config.kinopoisk.api.hash) }) req = self.pool.request(method or 'GET', url, headers=h, redirect=follow_redirects, body=data) if url_cache != None: content_type = req.getheader('Content-Type', '') if self._core.plugin_class == 'Agent' and not _content_type_allowed( content_type): self._core.log.debug( "Not caching '%s' (content type '%s' not cacheable in Agent plug-ins)", url, content_type) else: url_cache['content'] = req.data url_cache.headers = dict(req.headers) return req
def __response(cls, data: typing.Any) -> urllib3.HTTPResponse: return urllib3.HTTPResponse(json.dumps(data).encode('utf-8'), headers=cls.json_content_type_headers)
def test_http_response_error(self): r = urllib3.HTTPResponse(status=400, reason="Bad Request", request_url="http://localhost") ex = HTTPResponseError(r) assert str(ex) == "400 Bad Request for url http://localhost"
async def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None) -> requests.Response: urlparts = urlparse(request.url) if isinstance(timeout, tuple): connect_timeout, read_timeout = timeout else: connect_timeout = timeout read_timeout = timeout connection = await self.manager.get_connection(url=urlparts, verify=verify, cert=cert, timeout=connect_timeout) target = urlparts.path if urlparts.query: target += "?" + urlparts.query headers = [("host", urlparts.netloc)] + list(request.headers.items()) message = h11.Request(method=request.method, target=target, headers=headers) await connection.send_event(message) if request.body: body = (_encode(request.body) if isinstance(request.body, str) else request.body) message = h11.Data(data=body) await connection.send_event(message) message = h11.EndOfMessage() await connection.send_event(message) status_code = 0 headers = [] reason = b"" buffer = io.BytesIO() while True: event = await connection.receive_event(read_timeout) event_type = type(event) if event_type is h11.Response: status_code = event.status_code headers = [(key.decode(), value.decode()) for key, value in event.headers] reason = event.reason elif event_type is h11.Data: buffer.write(event.data) elif event_type is h11.EndOfMessage: buffer.seek(0) break await connection.close() resp = urllib3.HTTPResponse( body=buffer, headers=headers, status=status_code, reason=reason, preload_content=False, ) return self.build_response(request, resp)
async def send( self, request: requests.PreparedRequest, *args: typing.Any, **kwargs: typing.Any ): urlparts = urlparse(request.url) hostname = urlparts.hostname port = urlparts.port if port is None: port = {'http': 80, 'https': 443}[urlparts.scheme] reader, writer = await asyncio.open_connection(hostname, port) conn = h11.Connection(our_role=h11.CLIENT) message = h11.Request( method=request.method, target=urlparts.path, headers=[('host', hostname)] + list(request.headers.items()) ) data = conn.send(message) writer.write(data) message = h11.EndOfMessage() data = conn.send(message) writer.write(data) status_code = 0 headers = [] reason = b'' buffer = io.BytesIO() while True: event = conn.next_event() event_type = type(event) if event_type is h11.NEED_DATA: data = await reader.read(2048) conn.receive_data(data) elif event_type is h11.Response: status_code = event.status_code headers = [(key.decode(), value.decode()) for key, value in event.headers] reason = event.reason elif event_type is h11.Data: buffer.write(event.data) elif event_type is h11.EndOfMessage: buffer.seek(0) break writer.close() await writer.wait_closed() resp = urllib3.HTTPResponse( body=buffer, headers=headers, status=status_code, reason=reason, preload_content=False ) return self.build_response(request, resp)
# # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import absolute_import import pytest import urllib3 from botocore.exceptions import ClientError from mock import patch import sagemaker OK_RESPONSE = urllib3.HTTPResponse() OK_RESPONSE.status = 200 BAD_RESPONSE = urllib3.HTTPResponse() BAD_RESPONSE.status = 502 @patch('sagemaker.local.image._SageMakerContainer.train', return_value="/some/path/to/model") @patch('sagemaker.local.local_session.LocalSession') def test_create_training_job(train, LocalSession): local_sagemaker_client = sagemaker.local.local_session.LocalSagemakerClient( ) instance_count = 2 image = "my-docker-image:1.0"
def _make_http_response(body): return urllib3.HTTPResponse(body=bytes(json.dumps(body), 'utf-8'))