def _parse_response(self, resp):
# decode the JSON body and check for errors
obj = json_decode(resp.body)
if isinstance(obj, list):
# check if there is an error in the list of dicts,
# raise the first error seen
for item in obj:
if 'error' in item:
raise relax_exception(
httpclient.HTTPError(
resp.code if item['error'] != 'not_found' else 404,
item['reason'], resp))
elif 'error' in obj:
raise relax_exception(
httpclient.HTTPError(resp.code, obj['reason'], resp))
elif 'rows' in obj:
# check if there is an error in the result rows,
# raise the first error seen
for row in obj['rows']:
if 'error' in row:
raise relax_exception(
httpclient.HTTPError(
resp.code if row['error'] != 'not_found' else 404,
row['error'], resp))
return obj
def _parse_response(self, resp):
# decode the JSON body and check for errors
obj = json_decode(resp.body)
def to_code(err):
return {'not_found': 404, 'conflict': 409}.get(err, 400)
if isinstance(obj, list):
# check if there is an error in the list of dicts,
# raise the first error seen
for item in obj:
if 'error' in item:
raise relax_exception(
httpclient.HTTPError(to_code(item['error']),
item['reason'], resp))
elif 'error' in obj:
raise relax_exception(
httpclient.HTTPError(resp.code, obj['reason'], resp))
elif 'rows' in obj:
# check if there is an error in the result rows,
# raise the first error seen
for row in obj['rows']:
if 'error' in row:
raise relax_exception(
httpclient.HTTPError(to_code(row['error']),
row['error'], resp))
return obj
def test_request_initialise_namespace_http_error(fetch, url, create_namespace,
error):
fetch.side_effect = [httpclient.HTTPError(404, 'Not Found')]
create_namespace.side_effect = httpclient.HTTPError(
500, 'Internal Server Error')
yield request(None, 'c8ab01', 'pay load')
assert create_namespace.call_count == 1
assert fetch.call_count == 2
assert error.called
def test_request_initialise_namespace_http_conflict(fetch, url,
create_namespace, warning):
fetch.side_effect = [httpclient.HTTPError(404, 'Not Found')]
create_namespace.side_effect = httpclient.HTTPError(
409, 'Namespace already exists')
yield request(None, 'c8ab01', 'pay load')
assert create_namespace.call_count == 1
assert fetch.call_count == 2
assert warning.called
def render_template(filename):
if os.path.isdir(os.path.join(template_root, filename)):
filename = os.path.join(filename, 'index_orig.html')
else:
filename = '%s.html' % filename
if any(filename.lstrip('/').startswith(p) for p in blacklist_templates):
raise httpclient.HTTPError(404)
try:
return template_lookup.get_template(filename).render()
except exceptions.TopLevelLookupException:
raise httpclient.HTTPError(404)
def test_request_uninitialised_namespace_then_invalid_query(
fetch, url, create_namespace):
fetch.side_effect = [
httpclient.HTTPError(404, 'Not Found'),
httpclient.HTTPError(500, 'Internal Server Error')
]
with pytest.raises(httpclient.HTTPError) as exc:
yield request(None, 'c8ab01', 'pay load')
assert create_namespace.call_count == 1
assert fetch.call_count == 2
assert exc.value.code == 500
def render_template(self, sFilename, **kwargs):
if os.path.isdir(os.path.join(self.template_root, sFilename)):
sFilename = os.path.join(sFilename, 'index.html')
else:
sFilename = '%s.html' % sFilename
if any(sFilename.lstrip('/').startswith(p) for p in self.blacklist_templates):
raise httpclient.HTTPError(404)
try:
return self.template_lookup.get_template(sFilename).render(**kwargs)
except exceptions.TopLevelLookupException:
raise httpclient.HTTPError(404)
def render_template(self, filename, **kwargs):
if os.path.isdir(os.path.join(settings.__template_dir__, filename)):
#比如说filename是`app`，而实际上`/templates/app`是个目录，则实际渲染的是`/templates/app/index.html`模板
filename = os.path.join(filename, 'index.html')
else:
#如果`app`所对应的`/templates/app`不是个目录，则渲染`/templates/app.html`
filename = '%s.html' % filename
if any(filename.lstrip('/').startswith(p) for p in settings.__blacklist_templates__):
#检测是否访问的是列入黑名单(__blacklist_templates__)的渲染目录，如果是的话，则raise error 404
raise httpclient.HTTPError(404)
try:
#如果一切正常，利用TemplateLookup进行模板渲染
return settings.__template_lookup__.get_template(filename).render(**kwargs)
except exceptions.TopLevelLookupException:
raise httpclient.HTTPError(404)
def test_execute_with_403(self):
message = 'Unit test message'
room = 'unit_room'
actor = hipchat.Message('Unit Test Action', {
'message': message,
'room': room
})
# Valid response test
response_dict = {
'error': {
'code': 403,
'type': 'Forbidden',
'message': 'Hit the rate limit'
}
}
response_body = json.dumps(response_dict)
http_response = httpclient.HTTPError(code=403, response=response_body)
with mock.patch.object(actor, '_get_http_client') as m:
m.return_value = FakeExceptionRaisingHTTPClientClass()
m.return_value.response_value = http_response
with self.assertRaises(exceptions.InvalidCredentials):
yield actor._execute()
def test_group_validation_errors(self):
fake_resp_body = mock.MagicMock(name='response_body')
fake_resp_body.body = json.dumps(
{
'request': {
'id': 'fake_id',
'url': '/fake',
'method': 'GET',
'timestamp': '2016-12-28T22:36:36.324Z',
},
'response': {
'errors': [
{'message': 'Cant create spot requests.',
'code': 'GENERAL_ERROR'},
{'message': 'AMI ami-16fc4976 with an...',
'code': 'UnsupportedOperation'},
]
},
}
)
source_exc = httpclient.HTTPError(
400, '400 Bad Request', fake_resp_body)
fake_exc = spotinst.SpotinstException(source_exc)
self.assertEquals(
('Spotinst Request ID (fake_id) GET /fake: GENERAL_ERROR: Cant '
'create spot requests., UnsupportedOperation: AMI ami-16fc4976 '
'with an...'),
str(fake_exc))
def test_post_with_bad_data(service_client, get_repositories, get_repository):
get_repositories.return_value = future_repositories
get_repository.return_value = future_repository
mock_response = Mock()
mock_response.body = '{"errors": [{"source_id_type": "", "message": "not supported asset id type"}]}'
mock_response.code = 400
exc = httpclient.HTTPError(400, response=mock_response)
service_client.return_value = make_future(MagicMock())
client = yield service_client()
endpoint = client.repository.repositories[''].search.offers
endpoint.post.side_effect = exc
handler = _create_offers_handler()
# MUT
handler.request.body = (
'[{"source_id":' +
'"https://openpermissions.org/s0/hub1/asset/exampleco/ExampleCoPictureID/1",'
+ '"source_id_type":""}]')
with pytest.raises(HTTPError) as excinfo:
handler.post().result()
assert excinfo.value.status_code == mock_response.code
assert excinfo.value.errors == json.loads(mock_response.body)
def test_execute_with_unknown_exception(self):
topic = 'Unit test topic'
room = 'unit_room'
actor = hipchat.Topic('Unit Test Action', {
'topic': topic,
'room': room
})
# Valid response test
response_dict = {
'error': {
'code': 123,
'type': 'Unknown',
'message': 'Auth token not found'
}
}
response_body = json.dumps(response_dict)
http_response = httpclient.HTTPError(code=123, response=response_body)
with mock.patch.object(actor, '_get_http_client') as m:
m.return_value = FakeExceptionRaisingHTTPClientClass()
m.return_value.response_value = http_response
with self.assertRaises(exceptions.RecoverableActorFailure):
yield actor._execute()
def test_execute_fail(self):
actor = misc.GenericHTTP('Unit Test Action',
{'url': 'http://example.com'})
error = httpclient.HTTPError(code=401, response={})
actor._fetch = mock_tornado(exc=error)
with self.assertRaises(exceptions.InvalidCredentials):
yield actor.execute()
def test_fetch_unexpected_failure_raises_exc_and_called_once(self):
# Wipe out the 'default' http error handling config for this test.
self.client._EXCEPTIONS = {httpclient.HTTPError: {}}
e = httpclient.HTTPError(300, 'Unexpected')
self.http_client_mock.fetch.side_effect = e
with self.assertRaises(httpclient.HTTPError):
yield self.client.fetch(url='http://foo.com', method='GET')
self.http_client_mock.fetch.assert_called_once()
def get(self, name):
if not name:
self.write('No params to read')
else:
path = name.split('/')
if (len(path) == 2):
user = name.split('/')[0]
arch = name.split('/')[1]
s = session()
q1 = s.query(User).filter(User.name == user)
if (q1.count() == 1):
q2 = s.query(File).filter(File.user_id == q1[0].id,
File.name == arch)
if (q2.count() > 0):
if (q2[0].expiration > datetime.date.today()):
_file_dir = os.path.abspath(
"") + "/uploads/" + user
_file_path = "%s/%s" % (_file_dir, arch)
if not arch or not os.path.exists(_file_path):
raise httpclient.HTTPError(404)
self.set_header('Content-Type',
'application/force-download')
self.set_header('Content-Disposition',
'attachment; filename=%s' % arch)
with open(_file_path, "rb") as f:
try:
while True:
_buffer = f.read(4096)
if _buffer:
self.write(_buffer)
else:
f.close()
self.finish()
return
except:
raise httpclient.HTTPError(404)
raise httpclient.HTTPError(500)
else:
self.write('expired link')
else:
self.write('invalid file')
else:
self.write('invalid user')
else:
self.write('invalids params')
def test_request_uninitialised_namespace_standalone_mode(
options, fetch, url, create_namespace):
options.standalone = True
fetch.side_effect = [httpclient.HTTPError(404, 'Not Found')]
with pytest.raises(httpclient.HTTPError) as exc:
yield request(None, 'c8ab01', 'pay load')
assert not create_namespace.called
assert fetch.call_count == 1
assert exc.value.code == 404
def http_fetch(url):
filename = url.split('/')[-3] + '.jpg'
with open(os.path.join(path, filename), 'wb') as f:
response = yield client.fetch(url, streaming_callback=f.write)
if response.code != 200:
raise httpclient.HTTPError('Request picture error: {}'.format(
response.reason))
else:
print('Download {} success'.format(filename))
def test__initialise_namespace_http_error(options, fetch):
options.url_repo_db = 'https://localhost'
options.repo_db_port = '8080'
options.repo_db_path = '/bigdata/namespace/'
fetch.side_effect = httpclient.HTTPError(500, 'Internal Server Error')
with pytest.raises(httpclient.HTTPError) as exc:
yield _initialise_namespace('foo')
assert exc.value.code == 500
def test_fetch_500_raises_exc_and_logs_no_password(self):
# Note: thjis test does not actually validate the logging output. It
# should, but I don't know how to do that. :)
e = httpclient.HTTPError(500, 'Failure')
self.http_client_mock.fetch.side_effect = e
with self.assertRaises(httpclient.HTTPError):
yield self.client.fetch(url='http://foo.com',
method='GET',
auth_username='******',
auth_password='******')
def _render(obj, key):
if not obj.get(key):
return
try:
obj[key] = self.jinja_env.from_string(obj[key]).render(
_cookies=_cookies, **env)
return True
except Exception as e:
log_error = 'The error occurred when rendering template {}: {} \\r\\n {}'.format(
key, obj[key], repr(e))
raise httpclient.HTTPError(500, log_error)
def func3():
response = Mock()
response.body = json.dumps({
"status":
400,
"errors": [{
"source": "onboarding",
"message": "missing asset_type"
}]
})
raise httpclient.HTTPError(400, 'doh', response)
def test_process_bogus_response(self):
content = b'Slow Down'
stream = io.BytesIO(content)
request = httpclient.HTTPRequest('/')
headers = httputil.HTTPHeaders(
{'x-amzn-RequestId': '3840c615-0503-4a53-a2f6-07afa795a5d6',
'Date': 'Tue, 06 Jun 2017 18:31:47 GMT'})
response = httpclient.HTTPResponse(request, 503, headers, stream)
error = httpclient.HTTPError(503, 'Bad Request', response)
with self.client_with_default_creds('s3') as obj:
self.assertEqual(obj._process_error(error), (False, None))
def test_fetch_wrapper_with_other_failure(self):
actor = rollbar.RollbarBase('Unit Test Action', {})
response_dict = {'err': 1, 'messsage': 'Something bad happened'}
response_body = json.dumps(response_dict)
http_response = httpclient.HTTPError(code=123, response=response_body)
with mock.patch.object(actor, '_get_http_client') as m:
m.return_value = FakeExceptionRaisingHTTPClientClass()
m.return_value.response_value = http_response
with self.assertRaises(exceptions.RecoverableActorFailure):
yield actor._fetch_wrapper('http://fake.com')
def test_fetch_wrapper_with_403(self):
actor = rollbar.RollbarBase('Unit Test Action', {})
response_dict = {'err': 1, 'messsage': 'access token not found: xxx'}
response_body = json.dumps(response_dict)
http_response = httpclient.HTTPError(code=403, response=response_body)
with mock.patch.object(actor, '_get_http_client') as m:
m.return_value = FakeExceptionRaisingHTTPClientClass()
m.return_value.response_value = http_response
with self.assertRaises(exceptions.InvalidCredentials):
yield actor._fetch_wrapper('http://fake.com')
def test_internal_server_error(self):
# A 500 error is returned if an HTTP error occurs during the remote
# request/response process.
error = httpclient.HTTPError(500, message='bad wolf')
with self.patch_http_client(error):
response = self.fetch('/base/remote-path/')
self.assertEqual(500, response.code)
self.assertEqual(
'Internal server error:\n'
'error fetching data from '
'https://api.example.com:17070/remote-path/: '
'HTTP 500: bad wolf', response.body)
self.assertEqual('Internal Server Error', response.reason)
def test_async_request_error(self):
client = httpclient.AsyncHTTPClient()
expectation = str(uuid.uuid4().hex)
url = self.get_url('/latest/meta-data/iam/'
'security-credentials/?role={}'.format(expectation))
with mock.patch('tornado_aws.config.INSTANCE_ENDPOINT', url):
obj = config.Authorization('default', client=client)
with mock.patch.object(obj._client, 'fetch') as fetch:
future = concurrent.Future()
future.set_exception(httpclient.HTTPError(599))
fetch.return_value = future
with self.assertRaises(httpclient.HTTPError):
yield obj._get_role_async()
def test_tornado_httpclient_errors(self):
with mock.patch('tornado.httpclient.AsyncHTTPClient.fetch') as fetch:
fetch.side_effect = httpclient.HTTPError(599)
response = yield self.mixin.http_fetch(self.get_url('/test'))
self.assertFalse(response.ok)
self.assertEqual(response.code, 599)
self.assertEqual(response.attempts, 3)
self.assertIsNone(response.body)
self.assertIsNone(response.headers)
self.assertIsNone(response.links)
self.assertIsNone(response.raw)
for e in response.exceptions:
self.assertIsInstance(e, httpclient.HTTPError)
def make_err_request(self, role, access_key, secret_key, token):
url = self.get_url(
'/latest/meta-data/iam/security-credentials/{}?role={}&access_'
'key={}&secret_key={}&token={}'.format(role, role, access_key,
secret_key, token))
with mock.patch('tornado_aws.config.INSTANCE_ENDPOINT', url):
client = httpclient.HTTPClient()
obj = config.Authorization('default', client=client)
with mock.patch.object(obj, '_get_role') as get_role:
get_role.return_value = role
with mock.patch.object(client, 'fetch') as fetch:
fetch.side_effect = httpclient.HTTPError(599)
obj.refresh()
return obj
def mock_auth_exception():
content = b'<?xml version="1.0" encoding="UTF-8"?>\n<Error>' \
b'<Code>ExpiredToken</Code><Message>The provided ' \
b'token has expired.</Message><RequestId>DA1C5F5' \
b'26B1A0EF2</RequestId><HostId>381a948d-a988-4e7' \
b'a-aefe-e27e82555bed</HostId></Error>'
stream = io.BytesIO(content)
request = httpclient.HTTPRequest('/')
headers = httputil.HTTPHeaders(
{'Content-Type': 'application/xml',
'Server': 'Server',
'X-Amz-Request-Id': '3840c615-0503-4a53-a2f6-07afa795a5d6',
'Date': 'Tue, 06 Jun 2017 18:31:47 GMT'})
response = httpclient.HTTPResponse(request, 400, headers, stream)
return httpclient.HTTPError(400, 'Bad Request', response)
def test_process_error_awz_creds(self):
content = b'{"__type": "MissingAuthenticationTokenException", ' \
b'"message": "Missing Authentication Token"}'
stream = io.BytesIO(content)
request = httpclient.HTTPRequest('/test')
headers = httputil.HTTPHeaders(
{'Content-Type': 'application/x-amz-json-1.1',
'Server': 'Server',
'X-Amz-Request-Id': '3840c615-0503-4a53-a2f6-07afa795a5d6',
'Date': 'Tue, 06 Jun 2017 18:31:47 GMT'})
response = httpclient.HTTPResponse(request, 400, headers, stream)
error = httpclient.HTTPError(400, 'Bad Request', response)
with self.client_with_default_creds('dynamodb') as obj:
result = obj._process_error(error)
self.assertTrue(result[0])
