forked from nicky-zs/tornado-src-comment
-
Notifications
You must be signed in to change notification settings - Fork 0
/
httpclient.py
364 lines (311 loc) · 14.7 KB
/
httpclient.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
# vim: fileencoding=utf-8
from __future__ import absolute_import, division, with_statement
import calendar
import email.utils
import httplib
import time
import weakref
from tornado.escape import utf8
from tornado import httputil
from tornado.ioloop import IOLoop
from tornado.util import import_object, bytes_type
class HTTPClient(object):
""" 阻塞式的HTTP客户端。使用ioloop+异步HTTP客户端实现,基本只是用于测试。 """
def __init__(self, async_client_class=None, **kwargs):
self._io_loop = IOLoop()
if async_client_class is None:
async_client_class = AsyncHTTPClient
self._async_client = async_client_class(self._io_loop, **kwargs)
self._response = None
self._closed = False
def __del__(self):
self.close()
def close(self):
if not self._closed:
self._async_client.close()
self._io_loop.close()
self._closed = True
def fetch(self, request, **kwargs):
def callback(response):
self._response = response
self._io_loop.stop()
self._async_client.fetch(request, callback, **kwargs)
self._io_loop.start()
response = self._response
self._response = None
response.rethrow()
return response
class AsyncHTTPClient(object):
""" 非阻塞的HTTP客户端。
它会根据指定的实现子类来创建一个实例,并且该实例会像单例一样被复用(每个IOLoop一个实例),除非使用force_instance=True。"""
_impl_class = None # 实现子类
_impl_kwargs = None
_DEFAULT_MAX_CLIENTS = 10
@classmethod
def _async_clients(cls): # 返回cls对象上的clients复用dict,若还没有则新建一个WeakKeyDictionary
assert cls is not AsyncHTTPClient, "should only be called on subclasses"
if not hasattr(cls, '_async_client_dict'):
cls._async_client_dict = weakref.WeakKeyDictionary()
return cls._async_client_dict
def __new__(cls, io_loop=None, max_clients=None, force_instance=False, **kwargs):
io_loop = io_loop or IOLoop.instance()
if cls is AsyncHTTPClient: # 这里是AsyncHTTPClient类直接被构造
if cls._impl_class is None:
from tornado.simple_httpclient import SimpleAsyncHTTPClient
AsyncHTTPClient._impl_class = SimpleAsyncHTTPClient # AsyncHTTPClient类默认使用SimpleAsyncHTTPClient实现
impl = AsyncHTTPClient._impl_class
else: # 这里是AsyncHTTPClient的子类被构造
impl = cls
if io_loop in impl._async_clients() and not force_instance: # 如果已有实例则直接返回
return impl._async_clients()[io_loop]
else:
instance = super(AsyncHTTPClient, cls).__new__(impl)
args = {}
if cls._impl_kwargs:
args.update(cls._impl_kwargs)
args.update(kwargs)
if max_clients is not None:
# max_clients is special because it may be passed positionally instead of by keyword
args["max_clients"] = max_clients
elif "max_clients" not in args:
args["max_clients"] = AsyncHTTPClient._DEFAULT_MAX_CLIENTS
instance.initialize(io_loop, **args) # 初始化函数
if not force_instance:
impl._async_clients()[io_loop] = instance
return instance
def close(self):
"""Destroys this http client, freeing any file descriptors used.
Not needed in normal use, but may be helpful in unittests that
create and destroy http clients. No other methods may be called
on the AsyncHTTPClient after close().
"""
if self._async_clients().get(self.io_loop) is self:
del self._async_clients()[self.io_loop]
def fetch(self, request, callback, **kwargs):
"""Executes a request, calling callback with an `HTTPResponse`.
The request may be either a string URL or an `HTTPRequest` object.
If it is a string, we construct an `HTTPRequest` using any additional
kwargs: ``HTTPRequest(request, **kwargs)``
If an error occurs during the fetch, the HTTPResponse given to the
callback has a non-None error attribute that contains the exception
encountered during the request. You can call response.rethrow() to
throw the exception (if any) in the callback.
"""
raise NotImplementedError()
@staticmethod
def configure(impl, **kwargs):
"""Configures the AsyncHTTPClient subclass to use.
AsyncHTTPClient() actually creates an instance of a subclass.
This method may be called with either a class object or the
fully-qualified name of such a class (or None to use the default,
SimpleAsyncHTTPClient)
If additional keyword arguments are given, they will be passed
to the constructor of each subclass instance created. The
keyword argument max_clients determines the maximum number of
simultaneous fetch() operations that can execute in parallel
on each IOLoop. Additional arguments may be supported depending
on the implementation class in use.
Example::
AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
"""
if isinstance(impl, (unicode, bytes_type)):
impl = import_object(impl)
if impl is not None and not issubclass(impl, AsyncHTTPClient):
raise ValueError("Invalid AsyncHTTPClient implementation")
AsyncHTTPClient._impl_class = impl
AsyncHTTPClient._impl_kwargs = kwargs
@staticmethod
def _save_configuration():
return (AsyncHTTPClient._impl_class, AsyncHTTPClient._impl_kwargs)
@staticmethod
def _restore_configuration(saved):
AsyncHTTPClient._impl_class = saved[0]
AsyncHTTPClient._impl_kwargs = saved[1]
class HTTPRequest(object):
"""HTTP client request object."""
def __init__(self, url, method="GET", headers=None, body=None,
auth_username=None, auth_password=None,
connect_timeout=20.0, request_timeout=20.0,
if_modified_since=None, follow_redirects=True,
max_redirects=5, user_agent=None, use_gzip=True,
network_interface=None, streaming_callback=None,
header_callback=None, prepare_curl_callback=None,
proxy_host=None, proxy_port=None, proxy_username=None,
proxy_password='', allow_nonstandard_methods=False,
validate_cert=True, ca_certs=None,
allow_ipv6=None,
client_key=None, client_cert=None):
"""Creates an `HTTPRequest`.
All parameters except `url` are optional.
:arg string url: URL to fetch
:arg string method: HTTP method, e.g. "GET" or "POST"
:arg headers: Additional HTTP headers to pass on the request
:type headers: `~tornado.httputil.HTTPHeaders` or `dict`
:arg string auth_username: Username for HTTP "Basic" authentication
:arg string auth_password: Password for HTTP "Basic" authentication
:arg float connect_timeout: Timeout for initial connection in seconds
:arg float request_timeout: Timeout for entire request in seconds
:arg datetime if_modified_since: Timestamp for ``If-Modified-Since``
header
:arg bool follow_redirects: Should redirects be followed automatically
or return the 3xx response?
:arg int max_redirects: Limit for `follow_redirects`
:arg string user_agent: String to send as ``User-Agent`` header
:arg bool use_gzip: Request gzip encoding from the server
:arg string network_interface: Network interface to use for request
:arg callable streaming_callback: If set, `streaming_callback` will
be run with each chunk of data as it is received, and
`~HTTPResponse.body` and `~HTTPResponse.buffer` will be empty in
the final response.
:arg callable header_callback: If set, `header_callback` will
be run with each header line as it is received, and
`~HTTPResponse.headers` will be empty in the final response.
:arg callable prepare_curl_callback: If set, will be called with
a `pycurl.Curl` object to allow the application to make additional
`setopt` calls.
:arg string proxy_host: HTTP proxy hostname. To use proxies,
`proxy_host` and `proxy_port` must be set; `proxy_username` and
`proxy_pass` are optional. Proxies are currently only support
with `curl_httpclient`.
:arg int proxy_port: HTTP proxy port
:arg string proxy_username: HTTP proxy username
:arg string proxy_password: HTTP proxy password
:arg bool allow_nonstandard_methods: Allow unknown values for `method`
argument?
:arg bool validate_cert: For HTTPS requests, validate the server's
certificate?
:arg string ca_certs: filename of CA certificates in PEM format,
or None to use defaults. Note that in `curl_httpclient`, if
any request uses a custom `ca_certs` file, they all must (they
don't have to all use the same `ca_certs`, but it's not possible
to mix requests with ca_certs and requests that use the defaults.
:arg bool allow_ipv6: Use IPv6 when available? Default is false in
`simple_httpclient` and true in `curl_httpclient`
:arg string client_key: Filename for client SSL key, if any
:arg string client_cert: Filename for client SSL certificate, if any
"""
if headers is None:
headers = httputil.HTTPHeaders()
if if_modified_since:
timestamp = calendar.timegm(if_modified_since.utctimetuple())
headers["If-Modified-Since"] = email.utils.formatdate(
timestamp, localtime=False, usegmt=True)
self.proxy_host = proxy_host
self.proxy_port = proxy_port
self.proxy_username = proxy_username
self.proxy_password = proxy_password
self.url = url
self.method = method
self.headers = headers
self.body = utf8(body)
self.auth_username = auth_username
self.auth_password = auth_password
self.connect_timeout = connect_timeout
self.request_timeout = request_timeout
self.follow_redirects = follow_redirects
self.max_redirects = max_redirects
self.user_agent = user_agent
self.use_gzip = use_gzip
self.network_interface = network_interface
self.streaming_callback = streaming_callback
self.header_callback = header_callback
self.prepare_curl_callback = prepare_curl_callback
self.allow_nonstandard_methods = allow_nonstandard_methods
self.validate_cert = validate_cert
self.ca_certs = ca_certs
self.allow_ipv6 = allow_ipv6
self.client_key = client_key
self.client_cert = client_cert
self.start_time = time.time()
class HTTPResponse(object):
"""HTTP Response object.
Attributes:
* request: HTTPRequest object
* code: numeric HTTP status code, e.g. 200 or 404
* headers: httputil.HTTPHeaders object
* buffer: cStringIO object for response body
* body: respose body as string (created on demand from self.buffer)
* error: Exception object, if any
* request_time: seconds from request start to finish
* time_info: dictionary of diagnostic timing information from the request.
Available data are subject to change, but currently uses timings
available from http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html,
plus 'queue', which is the delay (if any) introduced by waiting for
a slot under AsyncHTTPClient's max_clients setting.
"""
def __init__(self, request, code, headers=None, buffer=None,
effective_url=None, error=None, request_time=None,
time_info=None):
self.request = request
self.code = code
if headers is not None:
self.headers = headers
else:
self.headers = httputil.HTTPHeaders()
self.buffer = buffer
self._body = None
if effective_url is None:
self.effective_url = request.url
else:
self.effective_url = effective_url
if error is None:
if self.code < 200 or self.code >= 300:
self.error = HTTPError(self.code, response=self)
else:
self.error = None
else:
self.error = error
self.request_time = request_time
self.time_info = time_info or {}
def _get_body(self):
if self.buffer is None:
return None
elif self._body is None:
self._body = self.buffer.getvalue()
return self._body
body = property(_get_body)
def rethrow(self):
"""If there was an error on the request, raise an `HTTPError`."""
if self.error:
raise self.error
def __repr__(self):
args = ",".join("%s=%r" % i for i in self.__dict__.iteritems())
return "%s(%s)" % (self.__class__.__name__, args)
class HTTPError(Exception):
"""Exception thrown for an unsuccessful HTTP request.
Attributes:
code - HTTP error integer error code, e.g. 404. Error code 599 is
used when no HTTP response was received, e.g. for a timeout.
response - HTTPResponse object, if any.
Note that if follow_redirects is False, redirects become HTTPErrors,
and you can look at error.response.headers['Location'] to see the
destination of the redirect.
"""
def __init__(self, code, message=None, response=None):
self.code = code
message = message or httplib.responses.get(code, "Unknown")
self.response = response
Exception.__init__(self, "HTTP %d: %s" % (self.code, message))
def main():
from tornado.options import define, options, parse_command_line
define("print_headers", type=bool, default=False)
define("print_body", type=bool, default=True)
define("follow_redirects", type=bool, default=True)
define("validate_cert", type=bool, default=True)
args = parse_command_line()
client = HTTPClient()
for arg in args:
try:
response = client.fetch(arg, follow_redirects=options.follow_redirects, validate_cert=options.validate_cert,)
except HTTPError, e:
if e.response is not None:
response = e.response
else:
raise
if options.print_headers:
print response.headers
if options.print_body:
print response.body
client.close()
if __name__ == "__main__":
main()