def get_resp_text(resp: Response, encoding=None):
"""提取Response的文本"""
if encoding:
resp.encoding = encoding
else:
resp.encoding = resp.apparent_encoding
return resp.text
def test_file_update():
account = Account.create_from_data(json.loads(helpers.account))
file_data = json.loads(helpers.file_data)
file_obj = File.create_from_data(file_data, parent_resource=account)
with patch('kloudless.resources.request') as mock_req:
resp = Response()
new_data = file_data.copy()
new_data['name'] = 'NewFileName'
resp._content = json.dumps(new_data).encode('utf-8')
resp.encoding = 'utf-8'
account_resp = Response()
account_resp._content = helpers.account.encode('utf-8')
account_resp.encoding = 'utf-8'
mock_req.side_effect = (resp, account_resp)
file_obj.name = 'NewFileName'
file_obj.parent_id = 'root'
file_obj.save()
expected_calls = [
# This is updating the file
call(file_obj._api_session.patch,
'accounts/%s/storage/files/%s' %
(account.id, file_data['id']),
params={},
data={
'name': u'NewFileName',
'parent_id': 'root'
},
configuration=file_obj._configuration),
# This is refreshing the parent resource
call(account._api_session.get,
'accounts/%s' % account.id,
configuration=account._configuration),
]
mock_req.assert_has_calls(expected_calls)
def get(self, uri, headers={}, timeout=None):
observation_datetime = self.conn.hget(uri, "observation_datetime")
module_logger.debug("received headers of {}".format(headers))
if observation_datetime is not None:
odt = datetime.datetime.strptime(observation_datetime.decode('utf-8'), "%Y-%m-%dT%H:%M:%S")
module_logger.debug("expiring URI if it more than {} seconds old".format(self.expiration_delta))
age = (datetime.datetime.utcnow() - odt).seconds
module_logger.debug("URI is {} - {} = {} seconds old".format(datetime.datetime.utcnow(), odt, age))
if age > self.expiration_delta:
self.purgeuri(uri)
if self.conn.hget(uri, "response_status") is None:
self.saveuri(uri, headers=headers)
req_headers = CaseInsensitiveDict(json.loads(self.conn.hget(uri, "request_headers")))
req_method = self.conn.hget(uri, "request_method")
module_logger.debug("generating request object for URI {} with headers {}".format(uri, req_headers))
request = requests.Request(req_method, uri, headers=req_headers)
request.prepare()
response = Response()
response.request = request
response.status_code = int(self.conn.hget(uri, "response_status"))
response.reason = self.conn.hget(uri, "response_reason")
response.elapsed = datetime.timedelta(microseconds=int(self.conn.hget(uri, "response_elapsed")))
if type(self.conn.hget(uri, "response_encoding")) is bytes:
response.encoding = self.conn.hget(uri, "response_encoding").decode('utf-8')
else:
response.encoding = self.conn.hget(uri, "response_encoding")
module_logger.debug("encoding set to {} for URI {}".format(response.encoding, uri))
response.headers = CaseInsensitiveDict(json.loads(self.conn.hget(uri, "response_headers")))
module_logger.debug("response headers pulled from caceh for URI {}: {}".format(
uri, response.headers
))
response._content = self.conn.hget(uri, "response_content")
response.url = uri
return response
def make_mock_response(filename, status_code=None):
response = Response()
response.status_code = status_code or 200
response.encoding = "utf-8"
with open(os.path.join(resource_location, filename)) as text:
response._content = text.read().encode()
return response
def request(method, url, **kwargs):
if 'data' in kwargs:
kwargs['params'] = kwargs.pop('data')
elif 'params' in kwargs and kwargs['params'] is None:
kwargs.pop('params')
auth = None
if 'auth' in kwargs:
auth = kwargs.pop('auth')
for i in ['auth', 'allow_redirects', 'stream']:
if i in kwargs:
kwargs.pop(i)
if app.app.registry.api_url in url:
if auth:
authorization = api.authorization
api.authorization = ('Basic', auth)
resp = api._gen_request(method.upper(), url, expect_errors=True, **kwargs)
if auth:
api.authorization = authorization
else:
resp = app._gen_request(method.upper(), url, expect_errors=True, **kwargs)
response = Response()
response.status_code = resp.status_int
response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
response.encoding = get_encoding_from_headers(response.headers)
response.raw = resp
response._content = resp.body
response.reason = resp.status
if isinstance(url, bytes):
response.url = url.decode('utf-8')
else:
response.url = url
response.request = resp.request
return response
def test_file_upload():
account = Account.create_from_data(json.loads(helpers.account))
file_data = json.loads(helpers.file_data)
with patch('kloudless.resources.request') as mock_req:
resp = Response()
resp._content = helpers.file_data.encode('utf-8')
resp.encoding = 'utf-8'
mock_req.return_value = resp
file_obj = File.create(parent_resource=account,
file_name=file_data['name'],
parent_id='root',
file_data=helpers.file_contents)
assert isinstance(file_obj, File)
for attr in ['id', 'name', 'type', 'size', 'account']:
assert file_data[attr] == getattr(file_obj, attr)
mock_req.assert_called_with(File._api_session.post,
'accounts/%s/storage/files' % account.id,
data=helpers.file_contents,
headers={
'Content-Type':
'application/octet-stream',
'X-Kloudless-Metadata':
json.dumps({
'name': file_data['name'],
'parent_id': 'root'
})
},
params=None,
configuration=None)
def build_response(self, req, resp):
"""Builds a :class:`Response <requests.Response>` object from a urllib3
response. This should not be called from user code, and is only exposed
for use when subclassing the
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`
:param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
:param resp: The urllib3 response object.
"""
response = Response()
# Fallback to None if there's no status_code, for whatever reason.
response.status_code = getattr(resp, 'status', None)
# Make headers case-insensitive.
response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
# Set encoding.
response.encoding = get_encoding_from_headers(response.headers)
response.raw = resp
response.reason = response.raw.reason
if isinstance(req.url, bytes):
response.url = req.url.decode('utf-8')
else:
response.url = req.url
# Add new cookies from the server.
extract_cookies_to_jar(response.cookies, req, resp)
# Give the Response some context.
response.request = req
response.connection = self
return response
def send(self, request, **kwargs):
url = urlparse(request.url)
if url.scheme != 'https':
raise Exception('Only HTTPS is supported!')
ctx = self._make_context()
conn = httpslib.HTTPSConnection(
url.hostname, url.port or 443, ssl_context=ctx)
conn.request(request.method, url.path, request.body, request.headers)
resp = conn.getresponse()
response = Response()
# Fallback to None if there's no status_code, for whatever reason.
response.status_code = getattr(resp, 'status', None)
# Make headers case-insensitive.
response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
# Set encoding.
response.encoding = get_encoding_from_headers(response.headers)
response.raw = resp
response.reason = response.raw.reason
if isinstance(request.url, bytes):
response.url = request.url.decode('utf-8')
else:
response.url = request.url
# Give the Response some context.
response.request = request
response.connection = self
return response
def build_response(self, req, resp):
"""Builds a :class:`Response <requests.Response>` object from a urllib3
response. This should not be called from user code, and is only exposed
for use when subclassing the
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`
:param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
:param resp: The urllib3 response object.
"""
response = Response()
# Fallback to None if there's no status_code, for whatever reason.
response.status_code = getattr(resp, 'status', None)
# Make headers case-insensitive.
response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
# Set encoding.
response.encoding = get_encoding_from_headers(response.headers)
response.raw = resp
response.reason = response.raw.reason
if isinstance(req.url, bytes):
response.url = req.url.decode('utf-8')
else:
response.url = req.url
# Add new cookies from the server.
extract_cookies_to_jar(response.cookies, req, resp)
# Give the Response some context.
response.request = req
response.connection = self
return response
def requests_call(self,
method,
url,
headers={},
params=None,
data=None,
stream=False):
if (self.MyServerURL == ''):
print('no target URL!')
return
print('request call: ' + method + ' ' + url)
headers.update(self.additional_headers)
mydata = {}
mydata['method'] = method
mydata['url'] = url
mydata['headers'] = headers
mydata['data'] = data
mydata['stream'] = stream
mydata['params'] = params
t = requests.post(self.MyServerURL, None, json.dumps(mydata))
t = json.loads(t.content)
final = Response()
final.status_code = t['statusCode']
final._content = bytes(t['body'], 'utf-8')
final.headers = t['headers']
final.encoding = 'utf-8'
tt = mydata
return (final)
def _receive_response(self, task, response):
"""
Called by the delegate when a response has been received.
This call is expected only on background threads, and thus may not do
anything that is not Python-thread-safe. This means that, for example,
it is safe to grab things from the _tasks dictionary, but it is not
safe to make other method calls on this object unless they explicitly
state that they are safe in background threads.
"""
queue, request = self._tasks[task]
resp = Response()
resp.status_code = getKey(response, 'statusCode')
resp.reason = ''
# TODO: Why do I have to do this?
raw_headers = getKey(response, 'allHeaderFields')
resp.headers = CaseInsensitiveDict(raw_headers)
resp.encoding = get_encoding_from_headers(resp.headers)
# TODO: This needs to point to an object that we can use to provide
# the various raw things that requests needs.
resp.raw = None
if isinstance(request.url, bytes):
resp.url = request.url.decode('utf-8')
else:
resp.url = request.url
resp.request = request
resp.connection = self
# Put this response on the queue.
queue.put_nowait(resp)
def _CreateSignResponse(code, status, wim_hash):
sign_resp = Response()
sign_resp.encoding = 'utf-8'
sign_resp._content = ('{"Status": "%s", "ErrorCode": 0, "SignedURL": '
'"%s", "Path": ""}' % (status, wim_hash)).encode()
sign_resp.status_code = code
return sign_resp
def send(self,
request: PreparedRequest,
stream=False,
timeout=None,
verify=True,
cert=None,
proxies=None):
request.url = request.url.decode("utf-8") if isinstance(
request.url, bytes) else request.url
# Check URL, ask the fake Icinga to handle it
self.check_path(request.url)
resp = self.icinga.handle(request)
# Create response, emulate equests.adapters.HTTPAdapter behavior a bit
response = Response()
response.status_code = resp.get("status_code", None)
response.headers = CaseInsensitiveDict(resp.get("headers", {}))
response.encoding = get_encoding_from_headers(response.headers)
response.reason = resp.get("reason", None)
response.url = request.url # Already decoded above
response.request = request
response.raw = StreamableBytesIO(resp.get("body", "").encode("utf-8"))
# Cookie jar is not mocked, as Icinga doesn't use cookies
# response.connection is not mocked, because it's not a response attribute by default
# Call settings hook with the settings used for this request (to make testing with these easier)
self.settings_hook(stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies)
return response
def _build_response_(self, with_errors=False):
r = Response()
r.status_code = 401
r.encoding = 'utf-8'
json = dumps(self._build_json_(with_errors))
r.raw = StringIO(json.encode())
return r
def bad_rs_request(method, url, **kwargs):
response = Response()
response.status_code = 403
response.encoding = "application/json"
response._content = '"Unauthorized: upload_view failed permission check"'
response.reason = "403 Forbidden"
return response
def http_response_to_response(self, http_response, prepared_request):
"""
transform a WSGIResponse into a requests's Response model
:param django.http.response.HttpResponse http_response: the http response send by django view
:return: the requests's Response model corresponding to the http_response
:rtype: Response
"""
response = Response()
# Fallback to None if there's no status_code, for whatever reason.
response.status_code = getattr(http_response, 'status_code', None)
# Make headers case-insensitive.
response.headers = CaseInsensitiveDict(getattr(http_response._headers, 'headers', {}))
# Set encoding.
response.encoding = get_encoding_from_headers(response.headers)
response.raw = http_response
response.reason = response.raw.reason_phrase
response._content = http_response.content
req = prepared_request
if isinstance(req.url, bytes): # pragma: no cover
response.url = req.url.decode('utf-8')
else:
response.url = req.url
# Add new cookies from the server.
extract_cookies_to_jar(response.cookies, req, response)
# Give the Response some context.
response.request = req
response.connection = self
return response
def make_mock_response(filename, status_code=None):
response = Response()
response.status_code = status_code or 200
response.encoding = "utf-8"
with open(os.path.join(resource_location, filename)) as text:
response._content = text.read().encode()
return response
def build_response(self, request, resp):
"""
Builds a Requests' response object. This emulates most of the logic of
the standard fuction but deals with the lack of the ``.headers``
property on the HTTP20Response object.
"""
response = Response()
response.status_code = resp.status
response.headers = CaseInsensitiveDict(resp.getheaders())
response.raw = resp
response.reason = resp.reason
response.encoding = get_encoding_from_headers(response.headers)
extract_cookies_to_jar(response.cookies, request, response)
response.url = request.url
response.request = request
response.connection = self
# One last horrible patch: Requests expects its raw responses to have a
# release_conn method, which I don't. We should monkeypatch a no-op on.
resp.release_conn = lambda: None
return response
def request(method, url, **kwargs):
response = Response()
response.status_code = 403
response.encoding = 'application/json'
response._content = '"Unauthorized: upload_view failed permission check"'
response.reason = '403 Forbidden'
return response
def create_response_object(status_code, text):
response = Response()
response.encoding = "utf-8"
response.status_code = status_code
response._content = bytes(text, response.encoding)
response.url = "mock_url"
return response
def test_iterate(self):
with patch.object(self.rsm, 'mk_request') as mock:
response = Response()
response.encoding = 'utf-8'
response._content = json.dumps([{
'uuid': 'person1',
'age': 1,
'name': 'person1'
}, {
'uuid': 'person2',
'age': 2,
'name': 'person2'
}])
mock.return_value = response
person1, person2 = self.rsm.iterate(TestPerson)
self.assertEqual(person1.uuid, 'person1')
self.assertEqual(person1.age, 1)
self.assertEqual(person1.name, 'person1')
self.assertTrue(person1.is_read_only())
self.assertEqual(person2.uuid, 'person2')
self.assertEqual(person2.age, 2)
self.assertEqual(person2.name, 'person2')
self.assertTrue(person2.is_read_only())
mock.assert_called_with(
'GET', 'http://www.example.org/repos/foo/%s.json' % (
fqcn(TestPerson),))
def get_objects(self,
accept=MEDIA_TYPE_STIX_V20,
start=0,
per_request=0,
**filter_kwargs):
self._verify_can_read()
query_params = _filter_kwargs_to_query_params(filter_kwargs)
assert isinstance(query_params, dict)
full_filter = BasicFilter(query_params)
objs = full_filter.process_filter(
self.objects,
("id", "type", "version"),
self.manifests,
100,
)[0]
if objs:
resp = Response()
resp.status_code = 200
resp.headers["Content-Range"] = f"items 0-{len(objs)}/{len(objs)}"
resp.encoding = "utf-8"
resp._content = bytes(
stix2.v20.Bundle(objects=objs).serialize(ensure_ascii=False),
resp.encoding)
return resp
else:
resp = Response()
resp.status_code = 404
resp.raise_for_status()
def return_response(self, method, path, data, headers, response):
action = headers.get('X-Amz-Target')
data = json.loads(to_str(data))
records = []
if action in (ACTION_CREATE_STREAM, ACTION_DELETE_STREAM):
event_type = (event_publisher.EVENT_KINESIS_CREATE_STREAM
if action == ACTION_CREATE_STREAM else
event_publisher.EVENT_KINESIS_DELETE_STREAM)
payload = {'n': event_publisher.get_hash(data.get('StreamName'))}
if action == ACTION_CREATE_STREAM:
payload['s'] = data.get('ShardCount')
event_publisher.fire_event(event_type, payload=payload)
elif action == ACTION_PUT_RECORD:
response_body = json.loads(to_str(response.content))
event_record = {
'data': data['Data'],
'partitionKey': data['PartitionKey'],
'sequenceNumber': response_body.get('SequenceNumber')
}
event_records = [event_record]
stream_name = data['StreamName']
lambda_api.process_kinesis_records(event_records, stream_name)
elif action == ACTION_PUT_RECORDS:
event_records = []
response_body = json.loads(to_str(response.content))
response_records = response_body['Records']
records = data['Records']
for index in range(0, len(records)):
record = records[index]
event_record = {
'data': record['Data'],
'partitionKey': record['PartitionKey'],
'sequenceNumber':
response_records[index].get('SequenceNumber')
}
event_records.append(event_record)
stream_name = data['StreamName']
lambda_api.process_kinesis_records(event_records, stream_name)
elif action == ACTION_UPDATE_SHARD_COUNT:
# Currently kinesalite, which backs the Kinesis implementation for localstack, does
# not support UpdateShardCount:
# https://github.com/mhart/kinesalite/issues/61
#
# [Terraform](https://www.terraform.io) makes the call to UpdateShardCount when it
# applies Kinesis resources. A Terraform run fails when this is not present.
#
# The code that follows just returns a successful response, bypassing the 400
# response that kinesalite returns.
#
response = Response()
response.status_code = 200
content = {
'CurrentShardCount': 1,
'StreamName': data['StreamName'],
'TargetShardCount': data['TargetShardCount']
}
response.encoding = 'UTF-8'
response._content = json.dumps(content)
return response
def build_response(self, request, resp):
"""
Builds a Requests' response object. This emulates most of the logic of
the standard fuction but deals with the lack of the ``.headers``
property on the HTTP20Response object.
"""
response = Response()
response.status_code = resp.status
response.headers = CaseInsensitiveDict(resp.getheaders())
response.raw = resp
response.reason = resp.reason
response.encoding = get_encoding_from_headers(response.headers)
extract_cookies_to_jar(response.cookies, request, response)
if isinstance(request.url, bytes):
response.url = request.url.decode('utf-8')
else:
response.url = request.url
response.request = request
response.connection = self
# One last horrible patch: Requests expects its raw responses to have a
# release_conn method, which I don't. We should monkeypatch a no-op on.
resp.release_conn = lambda: None
return response
def test_folder_creation():
account = Account.create_from_data(json.loads(helpers.account))
folder_data = json.loads(helpers.folder_data)
with patch('kloudless.resources.request') as mock_req:
resp = Response()
resp.status_code = 201
resp._content = helpers.folder_data.encode('utf-8')
resp.encoding = 'utf-8'
mock_req.return_value = resp
folder = Folder.create(parent_resource=account,
data={
'name': "TestFolder",
'parent_id': "root"
})
assert isinstance(folder, Folder)
for attr in ['id', 'name', 'type', 'size', 'account']:
assert folder_data[attr] == getattr(folder, attr)
mock_req.assert_called_with(Folder._api_session.post,
'accounts/%s/storage/folders' % account.id,
configuration=None,
params={},
data={
'name': 'TestFolder',
'parent_id': 'root'
})
async def wrap_async(response: ClientResponse) -> Response:
"""Build a ``requests`` response from a ``aiohttp`` response.
A ``requests.Response`` instance is built to provide synchronous
access to the original response's data. Note that the returned
response does not have proper data for :attr:``elapsed`` or
:attr:``request``.
The response will be consumed if it has not already.
"""
# Ensure the response data is read so that the wrapped response
# does not require any async methods.
await response.read()
wrapped = Response()
wrapped._content = response._body # type: ignore
wrapped._content_consumed = True # type: ignore
wrapped.status_code = response.status
wrapped.headers = CaseInsensitiveDict(response.headers)
wrapped.url = str(response.url) # `aiohttp` uses a `URL` object.
wrapped.encoding = response.get_encoding()
wrapped.history = [await wrap_async(rsp) for rsp in response.history]
wrapped.reason = response.reason or ""
wrapped.cookies = cookiejar_from_dict(response.cookies)
return wrapped
def generate_fake_error_response(msg, status_code=401, encoding='utf-8'):
r = Response()
r.status_code = status_code
r.encoding = encoding
r.raw = RequestsStringIO(msg.encode())
r._content_consumed = True
r._content = r.raw.read()
return r
def deserialize_response(serialized):
r = Response()
r.encoding = serialized['body']['encoding']
r.headers = CaseInsensitiveDict(serialized['headers'])
r.url = serialized.get('url', '')
r.status_code = serialized['status_code']
add_urllib3_response(serialized, r)
return r
def generate_fake_error_response(msg, status_code=401, encoding='utf-8'):
r = Response()
r.status_code = status_code
r.encoding = encoding
r.raw = RequestsStringIO(msg)
r._content_consumed = True
r._content = r.raw.read()
return r
def generate_fake_error_response(msg, status_code=401, encoding="utf-8"):
"""Generate a fake Response from requests."""
r = Response()
r.status_code = status_code
r.encoding = encoding
r.raw = RequestsStringIO(msg.encode())
r._content_consumed = True
r._content = r.raw.read()
return r
def request(method, url, **kwargs):
response = Response()
if method == 'POST' and '/upload' in url:
url = test.generate_docservice_url()
response.status_code = 200
response.encoding = 'application/json'
response._content = '{{"data":{{"url":"{url}","hash":"md5:{md5}","format":"application/msword","title":"name.doc"}},"get_url":"{url}"}}'.format(url=url, md5='0'*32)
response.reason = '200 OK'
return response
def generate_fake_error_response(msg, status_code=401, encoding="utf-8"):
"""Generate a fake Response from requests."""
r = Response()
r.status_code = status_code
r.encoding = encoding
r.raw = RequestsStringIO(msg.encode())
r._content_consumed = True
r._content = r.raw.read()
return r
def request(method, url, **kwargs):
response = Response()
if method == 'POST' and '/upload' in url:
url = test.generate_docservice_url()
response.status_code = 200
response.encoding = 'application/json'
response._content = '{{"data":{{"url":"{url}","hash":"md5:{md5}","format":"application/msword","title":"name.doc"}},"get_url":"{url}"}}'.format(url=url, md5='0'*32)
response.reason = '200 OK'
return response
def test_account_list():
resp = Response()
resp._content = helpers.account_list.encode('utf-8')
resp.encoding = 'utf-8'
with patch('kloudless.resources.request') as mock_req:
mock_req.return_value = resp
accounts = Account().all()
assert len(accounts) > 0
assert all([isinstance(x, Account) for x in accounts])
def send(self, request, **kwargs):
response = Response()
response.status_code = 200
response.reason = 'OK'
response.headers = {
'Content-Type': 'text/html; charset=UTF-8',
}
response.encoding = get_encoding_from_headers(response.headers)
response.raw = BytesIO(b'<!doctype html><html>Hello</html>')
return response
def build_response(req, delegate, cookiestore):
# type: (PreparedRequest, NSURLSessionAdapterDelegate, NSHTTPCookieStorage) -> Response
"""Build a requests `Response` object from the response data collected by the NSURLSessionDelegate, and the
default cookie store."""
response = Response()
# Fallback to None if there's no status_code, for whatever reason.
response.status_code = getattr(delegate, 'status', None)
# Make headers case-insensitive.
response.headers = CaseInsensitiveDict(getattr(delegate, 'headers', {}))
# Set encoding.
response.encoding = get_encoding_from_headers(response.headers)
# response.raw = resp
# response.reason = response.raw.reason
if isinstance(req.url, bytes):
response.url = req.url.decode('utf-8')
else:
response.url = req.url
# Add new cookies from the server. For NSURLSession these have already been parsed.
# jar = RequestsCookieJar()
# for cookie in cookiestore.cookies():
# print cookie
# c = Cookie(
# version=cookie.version(),
# name=cookie.name(),
# value=cookie.value(),
# port=8444,
# # port=cookie.portList()[-1],
# port_specified=0,
# domain=cookie.domain(),
# domain_specified=cookie.domain(),
# domain_initial_dot=cookie.domain(),
# path=cookie.path(),
# path_specified=cookie.path(),
# secure=!cookie.HTTPOnly(),
# expires=cookie.expiresDate(),
# comment=cookie.comment(),
# comment_url=cookie.commentURL(),
# )
# jar.set_cookie(c)
#
# response.cookies = jar
response.raw = io.BytesIO(buffer(delegate.output))
# Give the Response some context.
response.request = req
# response.connection = self
return response
def setUp(self):
self.cassette = cassette.Cassette(
TestCassette.cassette_name,
'json',
'w+'
)
r = Response()
r.status_code = 200
r.encoding = 'utf-8'
r.headers = CaseInsensitiveDict({'Content-Type': decode('foo')})
r.url = 'http://example.com'
cassette.add_urllib3_response({
'body': {
'string': decode('foo'),
'encoding': 'utf-8'
}
}, r)
self.response = r
r = Request()
r.method = 'GET'
r.url = 'http://example.com'
r.headers = {}
r.data = {'key': 'value'}
self.response.request = r.prepare()
self.response.request.headers.update(
{'User-Agent': 'betamax/test header'}
)
self.json = {
'request': {
'body': 'key=value',
'headers': {
'User-Agent': 'betamax/test header',
'Content-Length': '9',
'Content-Type': 'application/x-www-form-urlencoded',
},
'method': 'GET',
'uri': 'http://example.com/',
},
'response': {
'body': {
'string': decode('foo'),
'encoding': 'utf-8',
},
'headers': {'Content-Type': decode('foo')},
'status_code': 200,
'url': 'http://example.com',
},
'recorded_at': '2013-08-31T00:00:00',
}
self.date = datetime(2013, 8, 31)
self.cassette.save_interaction(self.response, self.response.request)
self.interaction = self.cassette.interactions[0]
self.interaction.recorded_at = self.date
def auth_wrapper(self, *args, **kwargs):
if hasattr(self, '_session') and self._session.auth:
return func(self, *args, **kwargs)
else:
from github3.models import GitHubError
# Mock a 401 response
r = Response()
r.status_code = 401
r.encoding = 'utf-8'
msg = ('{"message": "Requires username/password '
'authentication"}').encode()
r.raw = StringIO(msg)
raise GitHubError(r)
def test_get_when_non_strict_json(downloader):
response = Response()
response.encoding = "UTF-8"
response.status_code = 200
response._content = '\n{"text":"This is\nmy text"}'.encode("UTF-8")
expected = {"text": "This is\nmy text"}
when(requests).get(...).thenReturn(response)
result = downloader.get("https://test.com")
assert expected == result
unstub()
def setUp(self):
# Make a new serializer to test with
self.test_serializer = Serializer()
serializers.serializer_registry["test"] = self.test_serializer
# Instantiate the cassette to test with
self.cassette = cassette.Cassette(TestCassette.cassette_name, "test", record_mode="once")
# Create a new object to serialize
r = Response()
r.status_code = 200
r.reason = "OK"
r.encoding = "utf-8"
r.headers = CaseInsensitiveDict({"Content-Type": decode("foo")})
r.url = "http://example.com"
util.add_urllib3_response({"body": {"string": decode("foo"), "encoding": "utf-8"}}, r)
self.response = r
# Create an associated request
r = Request()
r.method = "GET"
r.url = "http://example.com"
r.headers = {}
r.data = {"key": "value"}
self.response.request = r.prepare()
self.response.request.headers.update({"User-Agent": "betamax/test header"})
# Expected serialized cassette data.
self.json = {
"request": {
"body": {"encoding": "utf-8", "string": "key=value"},
"headers": {
"User-Agent": ["betamax/test header"],
"Content-Length": ["9"],
"Content-Type": ["application/x-www-form-urlencoded"],
},
"method": "GET",
"uri": "http://example.com/",
},
"response": {
"body": {"string": decode("foo"), "encoding": "utf-8"},
"headers": {"Content-Type": [decode("foo")]},
"status": {"code": 200, "message": "OK"},
"url": "http://example.com",
},
"recorded_at": "2013-08-31T00:00:00",
}
self.date = datetime(2013, 8, 31)
self.cassette.save_interaction(self.response, self.response.request)
self.interaction = self.cassette.interactions[0]
self.interaction.recorded_at = self.date
def deserialize_response(serialized):
r = Response()
r.encoding = serialized['body']['encoding']
h = [(k, from_list(v)) for k, v in serialized['headers'].items()]
r.headers = CaseInsensitiveDict(h)
r.url = serialized.get('url', '')
if 'status' in serialized:
r.status_code = serialized['status']['code']
r.reason = serialized['status']['message']
else:
r.status_code = serialized['status_code']
r.reason = _codes[r.status_code][0].upper()
add_urllib3_response(serialized, r)
return r
def deserialize_response(serialized):
r = Response()
r.encoding = serialized['body']['encoding']
h = [(k, from_list(v)) for k, v in serialized['headers'].items()]
r.headers = CaseInsensitiveDict(h)
r.url = serialized.get('url', '')
if 'status' in serialized:
r.status_code = serialized['status']['code']
r.reason = serialized['status']['message']
else:
r.status_code = serialized['status_code']
r.reason = _codes[r.status_code][0].upper()
add_urllib3_response(serialized, r)
return r
def test_get_should_invalid_cache_on_too_many_redirects_error(mocker):
delete_cache = mocker.patch("cachecontrol.caches.file_cache.FileCache.delete")
response = Response()
response.encoding = "utf-8"
response.raw = BytesIO(encode('{"foo": "bar"}'))
mocker.patch(
"cachecontrol.adapter.CacheControlAdapter.send",
side_effect=[TooManyRedirects(), response],
)
repository = PyPiRepository()
repository._get("https://pypi.org/pypi/async-timeout/json")
assert delete_cache.called
def request(method, url, **kwargs):
response = Response()
if method == "POST" and "/upload" in url:
url = test.generate_docservice_url()
response.status_code = 200
response.encoding = "application/json"
data = '{{"url":"{url}","hash":"md5:{md5}","format":"{format}","title":"{title}"}}'.format(
url=url,
md5="0" * 32,
title="name.doc",
format="application/msword")
response._content = '{{"data": {data},"get_url":"{url}"}}'.format(
url=url, data=data)
response.reason = "200 OK"
return response
def test_get_should_invalid_cache_on_too_many_redirects_error(mocker: MockerFixture):
delete_cache = mocker.patch("cachecontrol.caches.file_cache.FileCache.delete")
response = Response()
response.status_code = 200
response.encoding = "utf-8"
response.raw = BytesIO(encode('{"foo": "bar"}'))
mocker.patch(
"poetry.utils.authenticator.Authenticator.get",
side_effect=[TooManyRedirects(), response],
)
repository = PyPiRepository()
repository._get("https://pypi.org/pypi/async-timeout/json")
assert delete_cache.called
def _lambda_decode_reponse(self, lambda_response):
"""
Convert json blob returned by lambda into one that requests
clients are used to.
"""
response = Response()
response.status_code = lambda_response.get("statusCode", 502)
response.reason = STATUS_CODES_TO_REASON_PHRASES[response.status_code]
response.headers = lambda_response.get("headers", {})
response.encoding = get_encoding_from_headers(response.headers)
if "body" in lambda_response:
response.raw = _decode_payload(lambda_response, "body")
elif "errorMessage" in lambda_response:
response.raw = _decode_payload(lambda_response, "errorMessage")
return response
def send(self, request, **kwargs):
response = Response()
response.headers = {}
response.encoding = 'utf-8' # FIXME: this is a complete guess
response.url = request.url
response.request = request
response.connection = self
try:
response.raw = open(request.url.replace('file://', ''), 'r')
except IOError as e:
response.status_code = 404
return response
response.status_code = 200
return response
def auth_wrapper(self, *args, **kwargs):
auth = False
if hasattr(self, '_session'):
auth = (self._session.auth or
self._session.headers.get('Authorization'))
if auth:
return func(self, *args, **kwargs)
else:
from github3.models import GitHubError
# Mock a 401 response
r = Response()
r.status_code = 401
r.encoding = 'utf-8'
r.raw = StringIO('{"message": "Requires authentication"}'.encode())
raise GitHubError(r)
def test_serialize_response(self):
r = Response()
r.status_code = 200
r.reason = "OK"
r.encoding = "utf-8"
r.headers = CaseInsensitiveDict()
r.url = "http://example.com"
util.add_urllib3_response({"body": {"string": decode("foo"), "encoding": "utf-8"}}, r)
serialized = util.serialize_response(r, False)
assert serialized is not None
assert serialized != {}
assert serialized["body"]["encoding"] == "utf-8"
assert serialized["body"]["string"] == "foo"
assert serialized["headers"] == {}
assert serialized["url"] == "http://example.com"
assert serialized["status"] == {"code": 200, "message": "OK"}
def test_get(self):
with patch.object(self.rsm, 'mk_request') as mock:
response = Response()
response.encoding = 'utf-8'
response._content = json.dumps({
'uuid': 'person1',
'age': 1,
'name': 'person1'
})
mock.return_value = response
person1 = self.rsm.get(TestPerson, 'person1')
self.assertEqual(person1.uuid, 'person1')
self.assertEqual(person1.age, 1)
self.assertEqual(person1.name, 'person1')
mock.assert_called_with(
'GET', 'http://www.example.org/repos/foo/%s/person1.json' % (
fqcn(TestPerson),))
def test_serialize_response(self):
r = Response()
r.status_code = 200
r.encoding = 'utf-8'
r.headers = CaseInsensitiveDict()
r.url = 'http://example.com'
cassette.add_urllib3_response({
'body': {
'string': decode('foo'),
'encoding': 'utf-8'
}
}, r)
serialized = cassette.serialize_response(r, 'json')
assert serialized is not None
assert serialized != {}
assert serialized['status_code'] == 200
assert serialized['body']['encoding'] == 'utf-8'
assert serialized['body']['string'] == 'foo'
assert serialized['headers'] == {}
assert serialized['url'] == 'http://example.com'
def retrieve(self, request):
filename = self._fn(request.url)
resp = Response()
headers = utils.read('%s.metadata' % filename)
if headers:
try:
headers = CaseInsensitiveDict(json.loads(headers))
except ValueError:
return None
headers['x-cache'] = 'HIT from %s' % self.__class__.__name__
resp.url = headers.pop('url', None)
resp.status_code = headers.pop('status-code', None)
resp.encoding = headers.pop('encoding', None)
resp.headers = headers
resp._content = utils.read(filename)
return resp
else:
return None
def test_serialize_response(self):
r = Response()
r.status_code = 200
r.reason = 'OK'
r.encoding = 'utf-8'
r.headers = CaseInsensitiveDict()
r.url = 'http://example.com'
util.add_urllib3_response({
'body': {
'string': decode('foo'),
'encoding': 'utf-8'
}
}, r, HTTPHeaderDict())
serialized = util.serialize_response(r, False)
assert serialized is not None
assert serialized != {}
assert serialized['body']['encoding'] == 'utf-8'
assert serialized['body']['string'] == 'foo'
assert serialized['headers'] == {}
assert serialized['url'] == 'http://example.com'
assert serialized['status'] == {'code': 200, 'message': 'OK'}
def deserialize_response(serialized):
r = Response()
r.encoding = serialized['body']['encoding']
header_dict = HTTPHeaderDict()
for header_name, header_list in serialized['headers'].items():
if isinstance(header_list, list):
for header_value in header_list:
header_dict.add(header_name, header_value)
else:
header_dict.add(header_name, header_list)
r.headers = CaseInsensitiveDict(header_dict)
r.url = serialized.get('url', '')
if 'status' in serialized:
r.status_code = serialized['status']['code']
r.reason = serialized['status']['message']
else:
r.status_code = serialized['status_code']
r.reason = _codes[r.status_code][0].upper()
add_urllib3_response(serialized, r, header_dict)
return r
def setUp(self):
# Make a new serializer to test with
self.test_serializer = Serializer()
serializers.serializer_registry['test'] = self.test_serializer
# Instantiate the cassette to test with
self.cassette = cassette.Cassette(
TestCassette.cassette_name,
'test',
record_mode='once'
)
# Create a new object to serialize
r = Response()
r.status_code = 200
r.reason = 'OK'
r.encoding = 'utf-8'
r.headers = CaseInsensitiveDict({'Content-Type': decode('foo')})
r.url = 'http://example.com'
util.add_urllib3_response({
'body': {
'string': decode('foo'),
'encoding': 'utf-8'
}
}, r, HTTPHeaderDict({'Content-Type': decode('foo')}))
self.response = r
# Create an associated request
r = Request()
r.method = 'GET'
r.url = 'http://example.com'
r.headers = {}
r.data = {'key': 'value'}
self.response.request = r.prepare()
self.response.request.headers.update(
{'User-Agent': 'betamax/test header'}
)
# Expected serialized cassette data.
self.json = {
'request': {
'body': {
'encoding': 'utf-8',
'string': 'key=value',
},
'headers': {
'User-Agent': ['betamax/test header'],
'Content-Length': ['9'],
'Content-Type': ['application/x-www-form-urlencoded'],
},
'method': 'GET',
'uri': 'http://example.com/',
},
'response': {
'body': {
'string': decode('foo'),
'encoding': 'utf-8',
},
'headers': {'Content-Type': [decode('foo')]},
'status': {'code': 200, 'message': 'OK'},
'url': 'http://example.com',
},
'recorded_at': '2013-08-31T00:00:00',
}
self.date = datetime(2013, 8, 31)
self.cassette.save_interaction(self.response, self.response.request)
self.interaction = self.cassette.interactions[0]
def build_response(self, request, resp):
"""
Builds a Requests' response object. This emulates most of the logic of
the standard fuction but deals with the lack of the ``.headers``
property on the HTTP20Response object.
Additionally, this function builds in a number of features that are
purely for HTTPie. This is to allow maximum compatibility with what
urllib3 does, so that HTTPie doesn't fall over when it uses us.
"""
response = Response()
response.status_code = resp.status
response.headers = CaseInsensitiveDict(resp.headers.iter_raw())
response.raw = resp
response.reason = resp.reason
response.encoding = get_encoding_from_headers(response.headers)
extract_cookies_to_jar(response.cookies, request, response)
response.url = request.url
response.request = request
response.connection = self
# First horrible patch: Requests expects its raw responses to have a
# release_conn method, which I don't. We should monkeypatch a no-op on.
resp.release_conn = lambda: None
# Next, add the things HTTPie needs. It needs the following things:
#
# - The `raw` object has a property called `_original_response` that is
# a `httplib` response object.
# - `raw._original_response` has three simple properties: `version`,
# `status`, `reason`.
# - `raw._original_response.version` has one of three values: `9`,
# `10`, `11`.
# - `raw._original_response.msg` exists.
# - `raw._original_response.msg._headers` exists and is an iterable of
# two-tuples.
#
# We fake this out. Most of this exists on our response object already,
# and the rest can be faked.
#
# All of this exists for httpie, which I don't have any tests for,
# so I'm not going to bother adding test coverage for it.
class FakeOriginalResponse(object): # pragma: no cover
def __init__(self, headers):
self._headers = headers
def get_all(self, name, default=None):
values = []
for n, v in self._headers:
if n == name.lower():
values.append(v)
if not values:
return default
return values
def getheaders(self, name):
return self.get_all(name, [])
response.raw._original_response = orig = FakeOriginalResponse(None)
orig.version = 20
orig.status = resp.status
orig.reason = resp.reason
orig.msg = FakeOriginalResponse(resp.headers.iter_raw())
return response
