def test_fetch(self):
# Test with valid JSON
response_dict = {'foo': 'asdf'}
response_body = json.dumps(response_dict)
http_response = httpclient.HTTPResponse(
httpclient.HTTPRequest('/'),
code=200,
buffer=io.StringIO(response_body))
with mock.patch.object(self.actor, '_get_http_client') as m:
m.return_value = FakeHTTPClientClass()
m.return_value.response_value = http_response
response = yield self.actor._fetch('/')
self.assertEqual(response_dict, response)
# Test with completely invalid JSON
response_body = "Something bad happened"
http_response = httpclient.HTTPResponse(
httpclient.HTTPRequest('/'),
code=200,
buffer=io.StringIO(response_body))
with mock.patch.object(self.actor, '_get_http_client') as m:
m.return_value = FakeHTTPClientClass()
m.return_value.response_value = http_response
with self.assertRaises(exceptions.UnparseableResponseFromEndpoint):
yield self.actor._fetch('/')
def test_execute_dry_mode_response(self):
topic = 'Unit test topic'
room = 'unit_room'
actor = hipchat.Topic('Unit Test Action', {
'topic': topic,
'room': room
})
# Valid response test
response_dict = {
'success': {
'code': 202,
'type': 'Accepted',
'message': 'It worked'
}
}
response_body = json.dumps(response_dict)
http_response = httpclient.HTTPResponse(
httpclient.HTTPRequest('/'),
code=202,
buffer=io.StringIO(response_body))
with mock.patch.object(actor, '_get_http_client') as m:
m.return_value = FakeHTTPClientClass()
m.return_value.response_value = http_response
res = yield actor._execute()
self.assertEqual(res, None)
def _CreateResponse(request):
print request
return httpclient.HTTPResponse(
request,
401,
headers={'Content-Type': 'application/json'},
buffer=None)
def make_response(code, body='', headers=None, request=None):
"""Create and return an HTTP response to be used in tests."""
if request is None:
request = httpclient.HTTPRequest('https://example.com')
return httpclient.HTTPResponse(request,
code,
headers=headers,
buffer=io.StringIO(body))
def fetch(self, request, *args, **kwargs):
future = concurrent.Future()
future.set_result(
httpclient.HTTPResponse(
request=request,
code=500,
error=ValueError('Expected programming error')))
return future
def mock_ok_response():
content = b'{"foo": "bar"}'
stream = io.BytesIO(content)
request = httpclient.HTTPRequest('/')
headers = httputil.HTTPHeaders(
{'x-amzn-RequestId': '3840c615-0503-4a53-a2f6-07afa795a5d6',
'Content-Type': 'application/x-amz-json-1.0',
'Server': 'Server',
'Date': 'Tue, 06 Jun 2017 18:31:47 GMT'})
return httpclient.HTTPResponse(request, 200, headers, stream)
def test_process_bogus_response(self):
content = b'Slow Down'
stream = io.BytesIO(content)
request = httpclient.HTTPRequest('/')
headers = httputil.HTTPHeaders(
{'x-amzn-RequestId': '3840c615-0503-4a53-a2f6-07afa795a5d6',
'Date': 'Tue, 06 Jun 2017 18:31:47 GMT'})
response = httpclient.HTTPResponse(request, 503, headers, stream)
error = httpclient.HTTPError(503, 'Bad Request', response)
with self.client_with_default_creds('s3') as obj:
self.assertEqual(obj._process_error(error), (False, None))
def test_response(self):
"""Map a url to a function returning an HTTPResponse.
HTTPResponse's constructor requires a request object, so there is no
fourth variant that returns a constant HTTPResponse.
"""
self.http_client.map(
kURL, lambda request: httpclient.HTTPResponse(
request, 404, buffer=StringIO("")))
self.http_client.fetch(kURL, self.stop)
response = self.wait()
self.assertEqual(response.code, 404)
def test_project(self):
actor = rollbar.RollbarBase('Unit Test Action', {})
response_dict = {'err': 0, 'result': '...'}
response_body = json.dumps(response_dict)
http_response = httpclient.HTTPResponse(
httpclient.HTTPRequest('/'), code=200,
buffer=StringIO.StringIO(response_body))
with mock.patch.object(actor, '_get_http_client') as m:
m.return_value = FakeHTTPClientClass()
m.return_value.response_value = http_response
res = yield actor._project()
self.assertEquals(res, response_dict)
def test_process_error_awz_creds(self):
content = b'{"__type": "MissingAuthenticationTokenException", ' \
b'"message": "Missing Authentication Token"}'
stream = io.BytesIO(content)
request = httpclient.HTTPRequest('/test')
headers = httputil.HTTPHeaders(
{'Content-Type': 'application/x-amz-json-1.1',
'Server': 'Server',
'X-Amz-Request-Id': '3840c615-0503-4a53-a2f6-07afa795a5d6',
'Date': 'Tue, 06 Jun 2017 18:31:47 GMT'})
response = httpclient.HTTPResponse(request, 400, headers, stream)
error = httpclient.HTTPError(400, 'Bad Request', response)
with self.client_with_default_creds('dynamodb') as obj:
result = obj._process_error(error)
self.assertTrue(result[0])
def test_fetch_wrapper(self):
actor = rollbar.RollbarBase('Unit Test Action', {})
# Valid response test
response_dict = {'status': 'sent'}
response_body = json.dumps(response_dict)
http_response = httpclient.HTTPResponse(
httpclient.HTTPRequest('/'), code=200,
buffer=StringIO.StringIO(response_body))
with mock.patch.object(actor, '_get_http_client') as m:
m.return_value = FakeHTTPClientClass()
m.return_value.response_value = http_response
res = yield actor._fetch_wrapper('http://fake.com')
self.assertEquals(res, response_dict)
def mock_auth_exception():
content = b'<?xml version="1.0" encoding="UTF-8"?>\n<Error>' \
b'<Code>ExpiredToken</Code><Message>The provided ' \
b'token has expired.</Message><RequestId>DA1C5F5' \
b'26B1A0EF2</RequestId><HostId>381a948d-a988-4e7' \
b'a-aefe-e27e82555bed</HostId></Error>'
stream = io.BytesIO(content)
request = httpclient.HTTPRequest('/')
headers = httputil.HTTPHeaders(
{'Content-Type': 'application/xml',
'Server': 'Server',
'X-Amz-Request-Id': '3840c615-0503-4a53-a2f6-07afa795a5d6',
'Date': 'Tue, 06 Jun 2017 18:31:47 GMT'})
response = httpclient.HTTPResponse(request, 400, headers, stream)
return httpclient.HTTPError(400, 'Bad Request', response)
def mock_error_exception():
content = b'<?xml version="1.0" encoding="UTF-8"?>' \
b'<Response><Errors><Error><Code>InvalidAction</Code>' \
b'<Message>The action CreateVolume is not valid for t' \
b'his web service.</Message></Error></Errors><Request' \
b'ID>cb159b66-84a7-43bf-8a02-c839cc50b164</RequestID>' \
b'</Response>'
stream = io.BytesIO(content)
request = httpclient.HTTPRequest('/')
headers = httputil.HTTPHeaders(
{'Content-Type': 'application/xml',
'Server': 'Server',
'X-Amz-Request-Id': '3840c615-0503-4a53-a2f6-07afa795a5d6',
'Date': 'Tue, 06 Jun 2017 18:31:47 GMT'})
response = httpclient.HTTPResponse(request, 400, headers, stream)
return httpclient.HTTPError(400, 'Bad Request', response)
def test_fetch_with_auth(self):
response_dict = {'foo': 'asdf'}
response_body = json.dumps(response_dict)
http_response = httpclient.HTTPResponse(
httpclient.HTTPRequest('/'),
code=200,
buffer=io.StringIO(response_body))
with mock.patch.object(self.actor, '_get_http_client') as m:
m.return_value = FakeHTTPClientClass()
m.return_value.response_value = http_response
yield self.actor._fetch('/',
auth_username='******',
auth_password='******')
self.assertEqual(m.return_value.request.auth_username, 'foo')
self.assertEqual(m.return_value.request.auth_password, 'bar')
def testAuthenticationFailed(self):
"""ERROR: Fail Facebook authentication (which returns None user_dict)."""
with mock.patch('tornado.httpclient.AsyncHTTPClient',
MockAsyncHTTPClient()) as mock_client:
mock_client.map(
r'https://graph.facebook.com/me\?',
lambda request: httpclient.HTTPResponse(request, 400))
url = self.get_url('/register/facebook?access_token=access_token')
self.assertRaisesHttpError(
401,
auth_test._SendAuthRequest,
self._tester,
url,
'POST',
user_cookie=self._cookie,
request_dict=auth_test._CreateRegisterRequest(
self._mobile_device_dict))
def test_process_simpledb_non_auth(self):
content = b'<?xml version="1.0" encoding="UTF-8"?>' \
b'<Response><Errors><Error><Code>InvalidAction</Code>' \
b'<Message>The action CreateVolume is not valid for t' \
b'his web service.</Message></Error></Errors><Request' \
b'ID>cb159b66-84a7-43bf-8a02-c839cc50b164</RequestID>' \
b'</Response>'
stream = io.BytesIO(content)
request = httpclient.HTTPRequest('/')
headers = httputil.HTTPHeaders(
{'x-amzn-RequestId': '3840c615-0503-4a53-a2f6-07afa795a5d6',
'Date': 'Tue, 06 Jun 2017 18:31:47 GMT'})
response = httpclient.HTTPResponse(request, 400, headers, stream)
error = httpclient.HTTPError(400, 'Bad Request', response)
with self.client_with_default_creds('dynamodb') as obj:
result = obj._process_error(error)
self.assertFalse(result[0])
self.assertIsInstance(result[1], exceptions.AWSError)
def test_process_error_s3(self):
content = b'<?xml version="1.0" encoding="UTF-8"?>\n<Error>' \
b'<Code>ExpiredToken</Code><Message>The provided ' \
b'token has expired.</Message><RequestId>DA1C5F5' \
b'26B1A0EF2</RequestId><HostId>381a948d-a988-4e7' \
b'a-aefe-e27e82555bed</HostId></Error>'
stream = io.BytesIO(content)
request = httpclient.HTTPRequest('/')
headers = httputil.HTTPHeaders(
{'Content-Type': 'application/xml',
'Server': 'Server',
'X-Amz-Request-Id': '3840c615-0503-4a53-a2f6-07afa795a5d6',
'Date': 'Tue, 06 Jun 2017 18:31:47 GMT'})
response = httpclient.HTTPResponse(request, 400, headers, stream)
error = httpclient.HTTPError(400, 'Bad Request', response)
with self.client_with_default_creds('s3') as obj:
result = obj._process_error(error)
self.assertTrue(result[0])
self.assertIsInstance(result[1], exceptions.AWSError)
def test_execute(self):
actor = librato.Annotation('Unit Test Action', {
'title': 'unittest',
'description': 'unittest',
'name': 'unittest'
})
response_dict = {'status': 'sent'}
response_body = json.dumps(response_dict)
http_response = httpclient.HTTPResponse(
httpclient.HTTPRequest('/'),
code=200,
buffer=StringIO.StringIO(response_body))
with mock.patch.object(actor, '_get_http_client') as m:
m.return_value = FakeHTTPClientClass()
m.return_value.response_value = http_response
res = yield actor._execute()
self.assertEquals(res, None)
def test_execute(self):
topic = 'Unit test topic'
room = 'unit_room'
actor = hipchat.Topic(
'Unit Test Action',
{'topic': topic, 'room': room})
# Valid response test
response_dict = {'status': 'sent'}
response_body = json.dumps(response_dict)
http_response = httpclient.HTTPResponse(
httpclient.HTTPRequest('/'), code=200,
buffer=StringIO.StringIO(response_body))
with mock.patch.object(actor, '_get_http_client') as m:
m.return_value = FakeHTTPClientClass()
m.return_value.response_value = http_response
res = yield actor._execute()
self.assertEquals(res, None)
def fetch(self, request, callback=None, **kwargs):
"""Implementation of AsyncHTTPClient.fetch"""
if not isinstance(request, httpclient.HTTPRequest):
request = httpclient.HTTPRequest(url=request, **kwargs)
for regex, response in self.url_map:
if regex.match(request.url):
if callable(response):
response = response(request)
if isinstance(response, basestring):
response = httpclient.HTTPResponse(
request, 200, buffer=StringIO(response))
assert isinstance(response, httpclient.HTTPResponse)
if callback is not None:
self.io_loop.add_callback(
functools.partial(callback, response))
return None
else:
future = Future()
future.set_result(response)
return future
raise ValueError("got request for unmapped url: %s" % request.url)
def _MakeResponse(max_index, request):
# Enforce maximum limit of 2.
request_dict = json.loads(request.body)
limit = min(request_dict.get('Limit', 2), 2)
is_count = request_dict.get('Count')
if 'ExclusiveStartKey' in request_dict:
start_index = int(request_dict['ExclusiveStartKey']
['RangeKeyElement']['S']['S'][-1]) + 1
else:
start_index = 0
count = min(max_index - start_index, limit)
items = []
for i in xrange(start_index, start_index + count):
items.append({'ei': {'S': 'e0'}, 'sk': {'S': 'p%d' % i}})
response_dict = {'Count': count, 'ConsumedCapacityUnits': 0.5}
if not is_count:
response_dict['Items'] = items
if start_index + count < max_index:
response_dict['LastEvaluatedKey'] = {
'HashKeyElement': {
'S': items[-1]['ei']
},
'RangeKeyElement': {
'S': items[-1]['sk']
}
}
return httpclient.HTTPResponse(
request,
200,
headers={'Content-Type': 'application/json'},
buffer=StringIO(json.dumps(response_dict)))
async def build_response(self,
obj,
proxy={},
CURL_ENCODING=config.curl_encoding,
CURL_CONTENT_LENGTH=config.curl_length,
EMPTY_RETRY=config.empty_retry):
try:
req, rule, env = self.build_request(
obj,
download_size_limit=self.download_size_limit,
proxy=proxy,
CURL_ENCODING=CURL_ENCODING,
CURL_CONTENT_LENGTH=CURL_CONTENT_LENGTH)
response = await gen.convert_yielded(self.client.fetch(req))
except httpclient.HTTPError as e:
try:
if config.allow_retry and pycurl:
if e.__dict__.get('errno', '') == 61:
logger.warning(
'{} {} [Warning] {} -> Try to retry!'.format(
req.method, req.url, e))
req, rule, env = self.build_request(
obj,
download_size_limit=self.download_size_limit,
proxy=proxy,
CURL_ENCODING=False,
CURL_CONTENT_LENGTH=CURL_CONTENT_LENGTH)
e.response = await gen.convert_yielded(
self.client.fetch(req))
elif e.code == 400 and e.message == 'Bad Request' and req and req.headers.get(
'content-length'):
logger.warning(
'{} {} [Warning] {} -> Try to retry!'.format(
req.method, req.url, e))
req, rule, env = self.build_request(
obj,
download_size_limit=self.download_size_limit,
proxy=proxy,
CURL_ENCODING=CURL_ENCODING,
CURL_CONTENT_LENGTH=False)
e.response = await gen.convert_yielded(
self.client.fetch(req))
elif e.code not in NOT_RETYR_CODE or (EMPTY_RETRY
and not e.response):
try:
logger.warning(
'{} {} [Warning] {} -> Try to retry!'.format(
req.method, req.url, e))
client = simple_httpclient.SimpleAsyncHTTPClient()
e.response = await gen.convert_yielded(
client.fetch(req))
except Exception:
logger.error(
e.message.replace('\\r\\n', '\r\n')
or e.response.replace('\\r\\n', '\r\n')
or Exception)
else:
try:
logger.warning('{} {} [Warning] {}'.format(
req.method, req.url, e))
except Exception:
logger.error(
e.message.replace('\\r\\n', '\r\n')
or e.response.replace('\\r\\n', '\r\n')
or Exception)
else:
logger.warning('{} {} [Warning] {}'.format(
req.method, req.url, e))
finally:
if 'req' not in locals().keys():
tmp = {'env': obj['env'], 'rule': obj['rule']}
tmp['request'] = {
'method': 'GET',
'url': 'http://127.0.0.1:8923/util/unicode?content=',
'headers': [],
'cookies': []
}
req, rule, env = self.build_request(tmp)
e.response = httpclient.HTTPResponse(request=req,
code=e.code,
reason=e.message,
buffer=BytesIO(
str(e).encode()))
if not e.response:
traceback.print_exc()
e.response = httpclient.HTTPResponse(request=req,
code=e.code,
reason=e.message,
buffer=BytesIO(
str(e).encode()))
return rule, env, e.response
return rule, env, response
def get_response(self, code=200, result=""):
request = httpclient.HTTPRequest('http://google.com')
return httpclient.HTTPResponse(request, code)
async def build_response(self,
obj,
proxy={},
CURL_ENCODING=config.curl_encoding,
CURL_CONTENT_LENGTH=config.curl_length,
EMPTY_RETRY=config.empty_retry):
try:
req, rule, env = self.build_request(
obj,
download_size_limit=self.download_size_limit,
proxy=proxy,
CURL_ENCODING=CURL_ENCODING,
CURL_CONTENT_LENGTH=CURL_CONTENT_LENGTH)
response = await gen.convert_yielded(self.client.fetch(req))
except httpclient.HTTPError as e:
try:
if config.allow_retry and pycurl:
if e.__dict__.get('errno', '') == 61:
logger.warning(
'{} {} [Warning] {} -> Try to retry!'.format(
req.method, req.url, e))
req, rule, env = self.build_request(
obj,
download_size_limit=self.download_size_limit,
proxy=proxy,
CURL_ENCODING=False,
CURL_CONTENT_LENGTH=CURL_CONTENT_LENGTH)
e.response = await gen.convert_yielded(
self.client.fetch(req))
elif e.code == 400 and e.message == 'Bad Request' and req and req.headers.get(
'content-length'):
logger.warning(
'{} {} [Warning] {} -> Try to retry!'.format(
req.method, req.url, e))
req, rule, env = self.build_request(
obj,
download_size_limit=self.download_size_limit,
proxy=proxy,
CURL_ENCODING=CURL_ENCODING,
CURL_CONTENT_LENGTH=False)
e.response = await gen.convert_yielded(
self.client.fetch(req))
elif e.code not in NOT_RETYR_CODE or (EMPTY_RETRY
and not e.response):
logger.warning(
'{} {} [Warning] {} -> Try to retry!'.format(
req.method, req.url, e))
client = simple_httpclient.SimpleAsyncHTTPClient()
e.response = await gen.convert_yielded(
client.fetch(req))
else:
logger.warning('{} {} [Warning] {}'.format(
req.method, req.url, e))
else:
logger.warning('{} {} [Warning] {}'.format(
req.method, req.url, e))
finally:
if not e.response:
traceback.print_exc()
from io import BytesIO
e.response = httpclient.HTTPResponse(request=req,
code=e.code,
reason=e.message,
buffer=BytesIO(
str(e).encode()))
return rule, env, e.response
return rule, env, response
def _CreateResponse(request):
return httpclient.HTTPResponse(
request,
201,
headers={'Content-Type': 'application/json'},
buffer=StringIO(json.dumps(response_dict)))
def fetch(self, url, method=None, body=None, callback=None, headers=None):
buf = io.StringIO(self.data_callback())
req = httpclient.HTTPRequest(url, method=method, headers=headers)
resp = httpclient.HTTPResponse(req, 200, buffer=buf)
return callback(resp)