class TestAWSConnection(unittest.TestCase):
@patch('patroni.scripts.aws.requests_get', Mock(return_value=urllib3.HTTPResponse(
status=200, body=b'{"instanceId": "012345", "region": "eu-west-1"}')))
def setUp(self):
self.conn = AWSConnection('test')
def test_on_role_change(self):
self.assertTrue(self.conn.on_role_change('master'))
with patch.object(MockVolumes, 'filter', Mock(return_value=[])):
self.conn._retry.max_tries = 1
self.assertFalse(self.conn.on_role_change('master'))
@patch('patroni.scripts.aws.requests_get', Mock(side_effect=Exception('foo')))
def test_non_aws(self):
conn = AWSConnection('test')
self.assertFalse(conn.on_role_change("master"))
@patch('patroni.scripts.aws.requests_get', Mock(return_value=urllib3.HTTPResponse(status=200, body=b'foo')))
def test_aws_bizare_response(self):
conn = AWSConnection('test')
self.assertFalse(conn.aws_available())
@patch('patroni.scripts.aws.requests_get', Mock(return_value=urllib3.HTTPResponse(status=503, body=b'Error')))
@patch('sys.exit', Mock())
def test_main(self):
self.assertIsNone(_main())
sys.argv = ['aws.py', 'on_start', 'replica', 'foo']
self.assertIsNone(_main())
def request(self, *args, **kwargs):
self._tc.assertTrue(len(self._reqs) > 0)
r = self._reqs.pop(0)
self._tc.maxDiff = None
self._tc.assertEqual(r[0], args)
self._tc.assertEqual(r[1], kwargs)
return urllib3.HTTPResponse(status=200, body=b'test')
def testQueryAPI(self):
"""Tests that query_api returns expected body."""
self.mock_http.request.return_value = urllib3.HTTPResponse(
body=b'{"expected": "body"}')
actual_response = self.deputy_instance.query_api(
'resource/Roster/QUERY', deputy.Query())
self.assertEqual(actual_response, {'expected': 'body'})
def test_request_with_headers(self):
# Given
connection = self.closing_http()
headers = {'Xtra Key': 'Xtra Value'}
http_response = urllib3.HTTPResponse(headers=headers)
request = self.patch('urllib3.PoolManager.request',
return_value=http_response)
retry = self.patch('urllib3.util.Retry')
# When
response, _ = connection.request(method=REQUEST_METHOD,
url=REQUEST_URL,
headers=headers)
# Then
request.assert_called_once_with(REQUEST_METHOD,
REQUEST_URL,
headers=dict(headers,
connection='close'),
retries=retry(raise_on_redirect=False,
redirect=5))
self.assertEqual(
{
'content-location': REQUEST_URL,
'status': str(http_response.status),
'xtra key': 'Xtra Value'
}, response)
def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None):
self.add_headers(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies)
# Default returns 200 and the same body message
status = 200
body = request.body or "No Body"
# check headers are present
if self.__checkHeaders(request.headers):
# Check signature is valid
timestamp = request.headers[self._protocol.TIMESTAMP_HEADER]
signature = self._protocol.signRequest(request.url, request.method, params=None, body=request.body, timestamp=timestamp)
signatureHeaders = self._protocol.getHeaders(signature)
for name, value in signatureHeaders.items():
_v = request.headers.get(name)
if not _v or _v != value:
status = 403
body = json.dumps({"error" : {"code" : 403, "message": "Headers {} is not correct".format(name)}})
break
else:
status = 403
body = json.dumps({"error" : {"code" : 403, "message": "Headers are missing"}})
headers = urllib3.response.HTTPHeaderDict(request.headers)
if hasattr(body, 'read'):
body.seek(0)
else:
body = BytesIO(body.encode('utf8'))
response = urllib3.HTTPResponse(body, headers, status, preload_content=False)
return self.build_response(request, response)
def test_request(self):
# Given
connection = self.closing_http()
http_response = urllib3.HTTPResponse()
request = self.patch('urllib3.PoolManager.request',
return_value=http_response)
retry = self.patch('urllib3.util.Retry')
# When
response, data = connection.request(method=REQUEST_METHOD,
url=REQUEST_URL)
# Then
request.assert_called_once_with(REQUEST_METHOD,
REQUEST_URL,
headers={'connection': 'close'},
retries=retry(raise_on_redirect=False,
redirect=5))
self.assertEqual(
{
'content-location': REQUEST_URL,
'status': str(http_response.status)
}, response)
self.assertEqual(http_response.status, response.status)
self.assertEqual(http_response.reason, response.reason)
self.assertEqual(http_response.version, response.version)
self.assertEqual(http_response.data, data)
def getLearningPlanData(request):
# Send the body of the data
data = {
"grant_type": "client_credentials",
"scope": {
"userId": "SFADMIN",
"companyId": "talenteam",
"userType": "user",
"resourceType": "learning_public_api"
}
}
headers = {'Authorization': '', 'Content-Type': 'application/json'}
url = 'https://talenteam-stage.plateau.com/learning/oauth-api/rest/v1/token'
response = requests.post(url, data=json.dumps(data), headers=headers)
data = response.json()
access_token = data['access_token']
if (access_token):
http = urllib3.PoolManager()
url = 'https://talenteam-stage.plateau.com/learning/odatav4/public/user/learningPlan/v1/UserTodoLearningItems?$filter=criteria/includeDeeplink eq true and criteria/includeLearnerActions eq true&$format=json&$select=title'
token = f"Bearer {access_token}"
print(token)
r = http.request('GET',
url,
headers={
'Authorization': token,
'Content-Type': 'application/json'
})
print(r.data)
return urllib3.HTTPResponse("Successful call")
def request(self, *actual_request_target, **actual_request_headers_and_body):
self._tc.assertTrue(len(self._reqs) > 0)
expected_results = self._reqs.pop(0)
self._tc.maxDiff = None
expected_request_target = expected_results[0] # The expected HTTP method and URL path.
expected_request_headers_and_body = expected_results[1] # dict that contains the expected body, headers
self._tc.assertEqual(expected_request_target, actual_request_target)
# actual_request_headers_and_body is a dict that contains the actual body, headers
for k, expected in expected_request_headers_and_body.items():
self._tc.assertIn(k, actual_request_headers_and_body)
if k == 'body':
actual_body = actual_request_headers_and_body[k]
self._tc.assertEqual(expected, actual_body)
elif k == 'headers':
actual_headers = actual_request_headers_and_body[k]
for expected_header_name, expected_header_value in expected.items():
# Validate the generated request contains the expected header.
self._tc.assertIn(expected_header_name, actual_headers)
actual_header_value = actual_headers[expected_header_name]
# Compare the actual value of the header against the expected value.
pattern = re.compile(expected_header_value)
m = pattern.match(actual_header_value)
self._tc.assertTrue(m, msg="Expected:\n{0}\nActual:\n{1}".format(
expected_header_value,actual_header_value))
if expected_header_name == 'Authorization':
self._validate_authorization_header(
expected_request_target, actual_headers, actual_header_value)
elif k == 'timeout':
self._tc.assertEqual(expected, actual_request_headers_and_body[k])
return urllib3.HTTPResponse(status=200, body=b'test')
class TestExhibitor(unittest.TestCase):
@patch('urllib3.PoolManager.request', Mock(return_value=urllib3.HTTPResponse(
status=200, body=b'{"servers":["127.0.0.1","127.0.0.2","127.0.0.3"],"port":2181}')))
@patch('patroni.dcs.zookeeper.PatroniKazooClient', MockKazooClient)
def setUp(self):
self.e = Exhibitor({'hosts': ['localhost', 'exhibitor'], 'port': 8181, 'scope': 'test',
'name': 'foo', 'ttl': 30, 'retry_timeout': 10})
@patch.object(ExhibitorEnsembleProvider, 'poll', Mock(return_value=True))
def test_get_cluster(self):
self.assertRaises(ZooKeeperError, self.e.get_cluster)
def testQueryAPI_PagesRequest(self, _, page_depth, result_length):
"""Tests that multiple requests are made when 500 records in response."""
long_response = []
for _ in range(0, 500):
long_response.append({'dummy': 'data'})
self.mock_http.request.side_effect = [
urllib3.HTTPResponse(
body=bytes(json.dumps(long_response), 'utf-8')),
urllib3.HTTPResponse(
body=bytes(json.dumps(long_response), 'utf-8')),
urllib3.HTTPResponse(
body=bytes(json.dumps({'dummy': 'data'}), 'utf-8'))
]
deputy_instance = deputy.Deputy(
'https://example.au.deputy.com/api/v1/', self.mock_http, '12345',
None, 10000, True, page_depth)
result = deputy_instance.query_api('resource/Roster/QUERY',
deputy.Query())
self.assertEqual(len(result), result_length)
def __response(cls,
body: typing.Union[str, bytes],
status: int = 200,
content_type: str = json_content_type,
headers: typing.Optional[dict[str, str]] = None,
preload_content: bool = True) -> urllib3.HTTPResponse:
if headers is None:
headers = {}
headers.update(cls.headers_for_content_type(content_type))
return urllib3.HTTPResponse(body,
headers=headers,
status=status,
preload_content=preload_content)
def test_get_recent_history_of_user(self):
with open(path.join(path.dirname(__file__),
'resource/recent_history.json'),
'r',
encoding='utf-8') as f:
try:
str = f.read()
except:
f.close()
response = bytes(str, encoding='utf-8')
r = urllib3.HTTPResponse(body=response)
urllib3.PoolManager.request = MagicMock(return_value=r)
result = asyncio.run(csgo_5e_query.get_recent_history_of_user('test'))
self.assertEqual(result[0]['rating'], "23.39")
def make_response(method,
url,
body=json.dumps({}),
headers={},
status=200):
from io import BytesIO
req = requests.Request(method.upper(), url).prepare()
body_stream = BytesIO(body) if isinstance(
body, bytes) else BytesIO(body.encode())
headers = urllib3.response.HTTPHeaderDict(headers)
resp = urllib3.HTTPResponse(body_stream,
headers,
status,
preload_content=False)
return requests.adapters.HTTPAdapter().build_response(req, resp)
def get_urlopen_mock(body=DEFAULT_BODY_CONTENT, headers=None, status=200):
mockHttpResponse = Mock(name='httplib.HTTPResponse')
headers = urllib3.response.HTTPHeaderDict(headers)
if not hasattr(body, 'read'):
body = BytesIO(body)
else:
body.seek(0)
urllib3_response = urllib3.HTTPResponse(body,
headers,
status,
preload_content=False,
original_response=mockHttpResponse)
return MagicMock(return_value=urllib3_response)
def test_request_with_fields(self):
# Given
connection = self.closing_http()
http_response = urllib3.HTTPResponse()
request = self.patch('urllib3.PoolManager.request',
return_value=http_response)
retry = self.patch('urllib3.util.Retry')
fields = object()
# When
connection.request(method=REQUEST_METHOD,
url=REQUEST_URL,
fields=fields)
# Then
request.assert_called_once_with(REQUEST_METHOD,
REQUEST_URL,
fields=fields,
headers=dict(connection='close'),
retries=retry(raise_on_redirect=False,
redirect=5))
def _submit_requests(self,
method,
url,
body=None,
headers=None,
query_params=None,
fields=None,
preload_content=None,
timeout=None,
encode_multipart=None):
if url.startswith('papi://PAPI_LOCAL_HOST'):
# operating on-cluster. use isi.rest with papi.sock instead of urllib3
# so we can avoid re-authenticating.
if 'Authorization' in headers:
del headers['Authorization']
if fields is not None:
# post parameters should be passed in the message body
body = fields
elif body is None:
# both body and fields are none, pass in an empty body
body = '{}'
if query_params is not None:
query_params = dict(query_params)
# the body field must be a string
if not isinstance(body, basestring):
body = json.dumps(body)
url_parts = url.partition('/platform/')[2].split('/')
# isi.rest will re-encode the url so decode each part. this is stupid
# but needed unless someone knows a way to get send_rest_request() to
# not re-encode the url parts.
for i in range(0, len(url_parts)):
url_parts[i] = unquote(url_parts[i])
response = isi.rest.send_rest_request(isi.rest.PAPI_SOCKET_PATH,
method=method,
uri=url_parts,
query_args=query_params,
headers=headers,
body=body,
timeout=120)
# we only use the headers, status, reason and body of a urllib3
# response object. the response from isi.rest.set_rest_request is:
# [status, headers subset, body]
reason = 'Response code: ' + str(response[0])
if 'status' in response[1]:
reason = response[1]['status']
# the contents of the status field are: 'XXX reason_string'
reason = reason[4:]
r = urllib3.HTTPResponse(headers=response[1],
status=response[0],
reason=reason,
body=response[2])
else:
if method in ['POST', 'PUT', 'PATCH', 'OPTIONS', 'DELETE']:
if query_params:
url += '?' + urlencode(query_params)
elif method in ['GET', 'HEAD']:
fields = query_params
if encode_multipart is not None:
r = self.pool_manager.request(
method,
url,
body=body,
fields=fields,
preload_content=preload_content,
timeout=timeout,
headers=headers,
encode_multipart=encode_multipart)
else:
r = self.pool_manager.request(method,
url,
body=body,
fields=fields,
preload_content=preload_content,
timeout=timeout,
headers=headers)
return r
async def send(
self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
) -> requests.Response:
urlparts = urlparse(request.url)
hostname = urlparts.hostname
port = urlparts.port
if port is None:
port = {"http": 80, "https": 443}[urlparts.scheme]
target = urlparts.path
if urlparts.query:
target += "?" + urlparts.query
headers = [("host", urlparts.netloc)] + list(request.headers.items())
conn_kwargs = {"ssl": no_verify()} if urlparts.scheme == "https" else {}
if isinstance(timeout, tuple):
connect_timeout, read_timeout = timeout
else:
connect_timeout = timeout
read_timeout = timeout
try:
reader, writer = await asyncio.wait_for(
asyncio.open_connection(hostname, port, **conn_kwargs), connect_timeout
)
except asyncio.TimeoutError:
raise requests.ConnectTimeout()
conn = h11.Connection(our_role=h11.CLIENT)
message = h11.Request(method=request.method, target=target, headers=headers)
data = conn.send(message)
writer.write(data)
if request.body:
message = h11.Data(data=request.body.encode("utf-8"))
data = conn.send(message)
writer.write(data)
message = h11.EndOfMessage()
data = conn.send(message)
writer.write(data)
status_code = 0
headers = []
reason = b""
buffer = io.BytesIO()
while True:
event = conn.next_event()
event_type = type(event)
if event_type is h11.NEED_DATA:
try:
data = await asyncio.wait_for(reader.read(2048), read_timeout)
except asyncio.TimeoutError:
raise requests.ReadTimeout()
conn.receive_data(data)
elif event_type is h11.Response:
status_code = event.status_code
headers = [
(key.decode(), value.decode()) for key, value in event.headers
]
reason = event.reason
elif event_type is h11.Data:
buffer.write(event.data)
elif event_type is h11.EndOfMessage:
buffer.seek(0)
break
writer.close()
if hasattr(writer, "wait_closed"):
await writer.wait_closed()
resp = urllib3.HTTPResponse(
body=buffer,
headers=headers,
status=status_code,
reason=reason,
preload_content=False,
)
return self.build_response(request, resp)
async def send(self, request: requests.PreparedRequest, *args: typing.Any,
**kwargs: typing.Any) -> requests.Response:
urlparts = urlparse(request.url)
hostname = urlparts.hostname
port = urlparts.port
if port is None:
port = {"http": 80, "https": 443}[urlparts.scheme]
target = urlparts.path
if urlparts.query:
target += "?" + urlparts.query
headers = [("host", urlparts.netloc)] + list(request.headers.items())
conn_kwargs = {
'ssl': no_verify()
} if urlparts.scheme == 'https' else {}
reader, writer = await asyncio.open_connection(hostname, port,
**conn_kwargs)
conn = h11.Connection(our_role=h11.CLIENT)
message = h11.Request(method=request.method,
target=target,
headers=headers)
data = conn.send(message)
writer.write(data)
if request.body:
message = h11.Data(data=request.body.encode("utf-8"))
data = conn.send(message)
writer.write(data)
message = h11.EndOfMessage()
data = conn.send(message)
writer.write(data)
status_code = 0
headers = []
reason = b""
buffer = io.BytesIO()
while True:
event = conn.next_event()
event_type = type(event)
if event_type is h11.NEED_DATA:
data = await reader.read(2048)
conn.receive_data(data)
elif event_type is h11.Response:
status_code = event.status_code
headers = [(key.decode(), value.decode())
for key, value in event.headers]
reason = event.reason
elif event_type is h11.Data:
buffer.write(event.data)
elif event_type is h11.EndOfMessage:
buffer.seek(0)
break
writer.close()
if hasattr(writer, 'wait_closed'):
await writer.wait_closed()
resp = urllib3.HTTPResponse(
body=buffer,
headers=headers,
status=status_code,
reason=reason,
preload_content=False,
)
return self.build_response(request, resp)
def urllib3_http_request(self,
url,
values=None,
headers={},
cacheTime=None,
encoding=None,
errors=None,
timeout=0,
immediate=False,
sleep=0,
data=None,
opener=None,
sandbox=None,
follow_redirects=True,
basic_auth=None,
method=None):
if cacheTime == None: cacheTime = self.cache_time
pos = url.rfind('#')
if pos > 0:
url = url[:pos]
if values and not data:
data = urllib.urlencode(values)
url_cache = None
if self._http_caching_enabled:
if cacheTime > 0:
cache_mgr = self._cache_mgr
if cache_mgr.item_count > self._core.config.http_cache_max_items + self._core.config.http_cache_max_items_grace:
cache_mgr.trim(self._core.config.http_cache_max_size,
self._core.config.http_cache_max_items)
url_cache = cache_mgr[url]
url_cache.set_expiry_interval(cacheTime)
else:
del self._cache_mgr[url]
if url_cache != None and url_cache['content'] and not url_cache.expired:
content_type = url_cache.headers.get('Content-Type', '')
if self._core.plugin_class == 'Agent' and not _content_type_allowed(
content_type):
self._core.log.debug(
"Removing cached data for '%s' (content type '%s' not cacheable in Agent plug-ins)",
url, content_type)
manager = url_cache._manager
del manager[url]
else:
self._core.log.debug("Fetching '%s' from the HTTP cache", url)
res = urllib3.HTTPResponse(url_cache['content'],
headers=url_cache.headers)
return res
h = dict(self.default_headers)
h.update({'connection': 'keep-alive'})
if sandbox:
h.update(sandbox.custom_headers)
h.update(headers)
if 'PLEXTOKEN' in os.environ and len(
os.environ['PLEXTOKEN']) > 0 and h is not None and url.find(
'http://127.0.0.1') == 0:
h['X-Plex-Token'] = os.environ['PLEXTOKEN']
if basic_auth != None:
h['Authorization'] = self.generate_basic_auth_header(*basic_auth)
if url.startswith(config.kinopoisk.api.base[:-2]):
h.update({
'clientDate': datetime.now().strftime("%H:%M %d.%m.%Y"),
'x-timestamp': str(int(round(time.time() * 1000)))
})
h.update({
'x-signature':
self._core.data.hashing.md5(
url[len(config.kinopoisk.api.base[:-2]):] +
h.get('x-timestamp') + config.kinopoisk.api.hash)
})
req = self.pool.request(method or 'GET',
url,
headers=h,
redirect=follow_redirects,
body=data)
if url_cache != None:
content_type = req.getheader('Content-Type', '')
if self._core.plugin_class == 'Agent' and not _content_type_allowed(
content_type):
self._core.log.debug(
"Not caching '%s' (content type '%s' not cacheable in Agent plug-ins)",
url, content_type)
else:
url_cache['content'] = req.data
url_cache.headers = dict(req.headers)
return req
def __response(cls, data: typing.Any) -> urllib3.HTTPResponse:
return urllib3.HTTPResponse(json.dumps(data).encode('utf-8'),
headers=cls.json_content_type_headers)
def test_http_response_error(self):
r = urllib3.HTTPResponse(status=400, reason="Bad Request", request_url="http://localhost")
ex = HTTPResponseError(r)
assert str(ex) == "400 Bad Request for url http://localhost"
async def send(self,
request,
stream=False,
timeout=None,
verify=True,
cert=None,
proxies=None) -> requests.Response:
urlparts = urlparse(request.url)
if isinstance(timeout, tuple):
connect_timeout, read_timeout = timeout
else:
connect_timeout = timeout
read_timeout = timeout
connection = await self.manager.get_connection(url=urlparts,
verify=verify,
cert=cert,
timeout=connect_timeout)
target = urlparts.path
if urlparts.query:
target += "?" + urlparts.query
headers = [("host", urlparts.netloc)] + list(request.headers.items())
message = h11.Request(method=request.method,
target=target,
headers=headers)
await connection.send_event(message)
if request.body:
body = (_encode(request.body)
if isinstance(request.body, str) else request.body)
message = h11.Data(data=body)
await connection.send_event(message)
message = h11.EndOfMessage()
await connection.send_event(message)
status_code = 0
headers = []
reason = b""
buffer = io.BytesIO()
while True:
event = await connection.receive_event(read_timeout)
event_type = type(event)
if event_type is h11.Response:
status_code = event.status_code
headers = [(key.decode(), value.decode())
for key, value in event.headers]
reason = event.reason
elif event_type is h11.Data:
buffer.write(event.data)
elif event_type is h11.EndOfMessage:
buffer.seek(0)
break
await connection.close()
resp = urllib3.HTTPResponse(
body=buffer,
headers=headers,
status=status_code,
reason=reason,
preload_content=False,
)
return self.build_response(request, resp)
async def send(
self, request: requests.PreparedRequest, *args: typing.Any, **kwargs: typing.Any
):
urlparts = urlparse(request.url)
hostname = urlparts.hostname
port = urlparts.port
if port is None:
port = {'http': 80, 'https': 443}[urlparts.scheme]
reader, writer = await asyncio.open_connection(hostname, port)
conn = h11.Connection(our_role=h11.CLIENT)
message = h11.Request(
method=request.method,
target=urlparts.path,
headers=[('host', hostname)] + list(request.headers.items())
)
data = conn.send(message)
writer.write(data)
message = h11.EndOfMessage()
data = conn.send(message)
writer.write(data)
status_code = 0
headers = []
reason = b''
buffer = io.BytesIO()
while True:
event = conn.next_event()
event_type = type(event)
if event_type is h11.NEED_DATA:
data = await reader.read(2048)
conn.receive_data(data)
elif event_type is h11.Response:
status_code = event.status_code
headers = [(key.decode(), value.decode()) for key, value in event.headers]
reason = event.reason
elif event_type is h11.Data:
buffer.write(event.data)
elif event_type is h11.EndOfMessage:
buffer.seek(0)
break
writer.close()
await writer.wait_closed()
resp = urllib3.HTTPResponse(
body=buffer,
headers=headers,
status=status_code,
reason=reason,
preload_content=False
)
return self.build_response(request, resp)
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
from __future__ import absolute_import
import pytest
import urllib3
from botocore.exceptions import ClientError
from mock import patch
import sagemaker
OK_RESPONSE = urllib3.HTTPResponse()
OK_RESPONSE.status = 200
BAD_RESPONSE = urllib3.HTTPResponse()
BAD_RESPONSE.status = 502
@patch('sagemaker.local.image._SageMakerContainer.train',
return_value="/some/path/to/model")
@patch('sagemaker.local.local_session.LocalSession')
def test_create_training_job(train, LocalSession):
local_sagemaker_client = sagemaker.local.local_session.LocalSagemakerClient(
)
instance_count = 2
image = "my-docker-image:1.0"
def _make_http_response(body):
return urllib3.HTTPResponse(body=bytes(json.dumps(body), 'utf-8'))
