def get_client(request):
if request.param == 'simple':
from tornado import simple_httpclient as simple
return (lambda: simple.SimpleAsyncHTTPClient())
elif request.param == 'curl':
curl = pytest.importorskip("tornado.curl_httpclient")
return (lambda: curl.CurlAsyncHTTPClient())
else:
return (lambda: http.AsyncHTTPClient())
async def worker() -> Tuple[int, float]:
count = 0
start = now = time.time()
session = simple_httpclient.SimpleAsyncHTTPClient()
while (now - start < duration):
resp = await session.fetch(url)
resp.body
count += 1
now = time.time()
return count, now - start
def get(self):
content = self.get_query_argument('chl', strip=False)
url = 'https://chart.googleapis.com/chart?cht=qr&chs=%dx%d&chl=%s&chld=|0'\
% (250, 250, escape.url_escape('ss://' + content, plus=False))
request = HTTPRequest(url)
if options.debug:
logging.info("qrcode url: " + url)
request.proxy_host = '127.0.0.1'
request.proxy_port = 8123
client = curl_httpclient.CurlAsyncHTTPClient()
else:
client = simple_httpclient.SimpleAsyncHTTPClient()
response = yield client.fetch(request)
self.write_png(response.body)
async def fetch_request(request, client=None, retry_callback=None, attempts=1):
if client is None:
client = simple_httpclient.SimpleAsyncHTTPClient()
if attempts <= 0:
raise ValueError('attempts should be > 0')
# Fail faster on connection if we can retry
request.connect_timeout = 5 if attempts > 1 else 20
# Save exceptions history for further analysis
except_history = []
while attempts:
wait_before_retry = 0
try:
resp = await client.fetch(request)
except httpclient.HTTPError as e:
# retry on s3 errors
if e.code == 500:
wait_before_retry = 0.2
elif e.code in (503, 599):
wait_before_retry = 1
else:
raise
except_history.append(e)
except socket.error as e:
# retry
except_history.append(e)
else:
return resp
attempts -= 1
if not attempts:
raise except_history[-1]
if wait_before_retry:
await asyncio.sleep(wait_before_retry)
if retry_callback:
await retry_callback(request, attempts)
async def build_response(self,
obj,
proxy={},
CURL_ENCODING=config.curl_encoding,
CURL_CONTENT_LENGTH=config.curl_length,
EMPTY_RETRY=config.empty_retry):
try:
req, rule, env = self.build_request(
obj,
download_size_limit=self.download_size_limit,
proxy=proxy,
CURL_ENCODING=CURL_ENCODING,
CURL_CONTENT_LENGTH=CURL_CONTENT_LENGTH)
response = await gen.convert_yielded(self.client.fetch(req))
except httpclient.HTTPError as e:
try:
if config.allow_retry and pycurl:
if e.__dict__.get('errno', '') == 61:
logger.warning(
'{} {} [Warning] {} -> Try to retry!'.format(
req.method, req.url, e))
req, rule, env = self.build_request(
obj,
download_size_limit=self.download_size_limit,
proxy=proxy,
CURL_ENCODING=False,
CURL_CONTENT_LENGTH=CURL_CONTENT_LENGTH)
e.response = await gen.convert_yielded(
self.client.fetch(req))
elif e.code == 400 and e.message == 'Bad Request' and req and req.headers.get(
'content-length'):
logger.warning(
'{} {} [Warning] {} -> Try to retry!'.format(
req.method, req.url, e))
req, rule, env = self.build_request(
obj,
download_size_limit=self.download_size_limit,
proxy=proxy,
CURL_ENCODING=CURL_ENCODING,
CURL_CONTENT_LENGTH=False)
e.response = await gen.convert_yielded(
self.client.fetch(req))
elif e.code not in NOT_RETYR_CODE or (EMPTY_RETRY
and not e.response):
try:
logger.warning(
'{} {} [Warning] {} -> Try to retry!'.format(
req.method, req.url, e))
client = simple_httpclient.SimpleAsyncHTTPClient()
e.response = await gen.convert_yielded(
client.fetch(req))
except Exception:
logger.error(
e.message.replace('\\r\\n', '\r\n')
or e.response.replace('\\r\\n', '\r\n')
or Exception)
else:
try:
logger.warning('{} {} [Warning] {}'.format(
req.method, req.url, e))
except Exception:
logger.error(
e.message.replace('\\r\\n', '\r\n')
or e.response.replace('\\r\\n', '\r\n')
or Exception)
else:
logger.warning('{} {} [Warning] {}'.format(
req.method, req.url, e))
finally:
if 'req' not in locals().keys():
tmp = {'env': obj['env'], 'rule': obj['rule']}
tmp['request'] = {
'method': 'GET',
'url': 'http://127.0.0.1:8923/util/unicode?content=',
'headers': [],
'cookies': []
}
req, rule, env = self.build_request(tmp)
e.response = httpclient.HTTPResponse(request=req,
code=e.code,
reason=e.message,
buffer=BytesIO(
str(e).encode()))
if not e.response:
traceback.print_exc()
e.response = httpclient.HTTPResponse(request=req,
code=e.code,
reason=e.message,
buffer=BytesIO(
str(e).encode()))
return rule, env, e.response
return rule, env, response
async def build_response(self,
obj,
proxy={},
CURL_ENCODING=config.curl_encoding,
CURL_CONTENT_LENGTH=config.curl_length,
EMPTY_RETRY=config.empty_retry):
try:
req, rule, env = self.build_request(
obj,
download_size_limit=self.download_size_limit,
proxy=proxy,
CURL_ENCODING=CURL_ENCODING,
CURL_CONTENT_LENGTH=CURL_CONTENT_LENGTH)
response = await gen.convert_yielded(self.client.fetch(req))
except httpclient.HTTPError as e:
try:
if config.allow_retry and pycurl:
if e.__dict__.get('errno', '') == 61:
logger.warning(
'{} {} [Warning] {} -> Try to retry!'.format(
req.method, req.url, e))
req, rule, env = self.build_request(
obj,
download_size_limit=self.download_size_limit,
proxy=proxy,
CURL_ENCODING=False,
CURL_CONTENT_LENGTH=CURL_CONTENT_LENGTH)
e.response = await gen.convert_yielded(
self.client.fetch(req))
elif e.code == 400 and e.message == 'Bad Request' and req and req.headers.get(
'content-length'):
logger.warning(
'{} {} [Warning] {} -> Try to retry!'.format(
req.method, req.url, e))
req, rule, env = self.build_request(
obj,
download_size_limit=self.download_size_limit,
proxy=proxy,
CURL_ENCODING=CURL_ENCODING,
CURL_CONTENT_LENGTH=False)
e.response = await gen.convert_yielded(
self.client.fetch(req))
elif e.code not in NOT_RETYR_CODE or (EMPTY_RETRY
and not e.response):
logger.warning(
'{} {} [Warning] {} -> Try to retry!'.format(
req.method, req.url, e))
client = simple_httpclient.SimpleAsyncHTTPClient()
e.response = await gen.convert_yielded(
client.fetch(req))
else:
logger.warning('{} {} [Warning] {}'.format(
req.method, req.url, e))
else:
logger.warning('{} {} [Warning] {}'.format(
req.method, req.url, e))
finally:
if not e.response:
traceback.print_exc()
from io import BytesIO
e.response = httpclient.HTTPResponse(request=req,
code=e.code,
reason=e.message,
buffer=BytesIO(
str(e).encode()))
return rule, env, e.response
return rule, env, response
