def get(self, distro=None, comp=None):
db = self.settings['db']
(expired, dt) = yield self._cache_expired('components', {'distro': distro, 'component': comp})
if not expired:
self.set_status(304)
return
if not dt:
self.set_status(404)
return
self.add_header('Last-Modified', httputil.format_timestamp(dt))
self.set_header('Content-Type', 'application/octet-stream')
doc = yield db.cacus.components.find_one({'distro': distro, 'component': comp})
if doc:
s = common.config['repo_daemon']
if s['proxy_storage']:
headers = [ ('Content-Length', doc['size']), ('Last-Modified', httputil.format_timestamp(dt)) ]
yield self.stream_from_storage(doc['sources_file'], headers=headers)
else:
# we use x-accel-redirect instead of direct proxying via storage plugin to allow
# user to offload cacus' StorageHandler if current storage allows it
url = os.path.join(s['repo_base'], s['storage_subdir'], doc['sources_file'])
app_log.info("Redirecting %s/%s/source/Sources to %s", distro, comp, arch, url)
self.add_header("X-Accel-Redirect", url)
self.set_status(200)
else:
self.set_status(404)
def test_static_if_modified_since_time_zone(self):
# Instead of the value from Last-Modified, make requests with times
# chosen just before and after the known modification time
# of the file to ensure that the right time zone is being used
# when parsing If-Modified-Since.
stat = os.stat(relpath('static/robots.txt'))
response = self.fetch('/static/robots.txt', headers={
'If-Modified-Since': format_timestamp(stat.st_mtime - 1)})
self.assertEqual(response.code, 200)
response = self.fetch('/static/robots.txt', headers={
'If-Modified-Since': format_timestamp(stat.st_mtime + 1)})
self.assertEqual(response.code, 304)
def rel_as_dict(self, rel) :
return {"uuid" : None if rel.uuid.startswith("pseudo:") else rel.uuid,
"date_created" : httputil.format_timestamp(rel.date_created),
"deleted" : getattr(rel, "deleted", None),
"name" : rel.name,
"subject" : rel.subject_uuid,
"object" : rel.object_uuid,
"payload" : rel.payload}
def set_cookie(self, name, value, expires_days=30, version=None,
domain=None, expires=None, path="/", **kwargs):
""" Sets the given cookie name/value with the given options. Set value
to None to clear. The cookie value is secured using
`flexx.config.cookie_secret`; don't forget to set that config
value in your server. Additional keyword arguments are set on
the Cookie.Morsel directly.
"""
# This code is taken (in modified form) from the Tornado project
# Copyright 2009 Facebook
# Licensed under the Apache License, Version 2.0
# Assume tornado is available ...
from tornado.escape import native_str
from tornado.httputil import format_timestamp
from tornado.web import create_signed_value
# Clear cookie?
if value is None:
value = ""
expires = datetime.datetime.utcnow() - datetime.timedelta(days=365)
else:
secret = config.cookie_secret
value = create_signed_value(secret, name, value, version=version,
key_version=None)
# The cookie library only accepts type str, in both python 2 and 3
name = native_str(name)
value = native_str(value)
if re.search(r"[\x00-\x20]", name + value):
# Don't let us accidentally inject bad stuff
raise ValueError("Invalid cookie %r: %r" % (name, value))
if name in self._cookies:
del self._cookies[name]
self._cookies[name] = value
morsel = self._cookies[name]
if domain:
morsel["domain"] = domain
if expires_days is not None and not expires:
expires = datetime.datetime.utcnow() + datetime.timedelta(
days=expires_days)
if expires:
morsel["expires"] = format_timestamp(expires)
if path:
morsel["path"] = path
for k, v in kwargs.items():
if k == 'max_age':
k = 'max-age'
# skip falsy values for httponly and secure flags because
# SimpleCookie sets them regardless
if k in ['httponly', 'secure'] and not v:
continue
morsel[k] = v
self._exec('document.cookie = "%s";' %
morsel.OutputString().replace('"', '\\"'))
def blob_as_dict(self, blob, with_content=False) :
ret = {"uuid" : blob.uuid,
"date_created" : httputil.format_timestamp(blob.date_created),
"editor_email" : blob.editor_email,
"content_type" : blob.content_type}
if blob.content_type.startswith("mime:text/") :
ret["summary"] = self.blob_summary(blob)
if with_content :
ret["content"] = blob.content.stuff
return ret
def set_default_headers(self):
default_headers = {
"Server": "TornadoServer/%s" % tornado.version,
"Content-Type": "text/event-stream",
"access-control-allow-origin": "*",
"connection": "keep-alive",
"Date": httputil.format_timestamp(time.time()),
}
default_headers.update(self.custom_headers())
self._headers = httputil.HTTPHeaders(default_headers)
def redirect_after(self, request):
"Perform a redirect to ``target``"
date = request.params.get("date")
if date:
retry_after = str(httputil.format_timestamp(datetime.fromtimestamp(float(date))))
else:
retry_after = "1"
target = request.params.get("target", "/")
headers = [("Location", target), ("Retry-After", retry_after)]
return Response(status="303 See Other", headers=headers)
def clear(self):
"""Resets all headers and content for this response."""
self._headers = httputil.HTTPHeaders({
"Server": "Durotar/%s" % durotar.version,
"Content-Type": "text/html; charset=UTF-8",
"Date": httputil.format_timestamp(time.time()),
})
self.set_default_headers()
self._write_buffer = []
self._status_code = 200
self._reason = httputil.responses[200]
def redirect_after(self, request):
"Perform a redirect to ``target``"
date = request.params.get('date')
if date:
retry_after = str(httputil.format_timestamp(
datetime.fromtimestamp(float(date))))
else:
retry_after = '1'
target = request.params.get('target', '/')
headers = [('Location', target), ('Retry-After', retry_after)]
return Response(status='303 See Other', headers=headers)
def assert_modified(self, url, mod_date):
response = self.fetch(
url, if_modified_since=(mod_date - timedelta(seconds=1)))
# 200 OK, not 304 Not Modified.
self.assertEqual(200, response.code)
self.assertEqual(
httputil.format_timestamp(mod_date),
response.headers['Last-Modified'])
response = self.fetch(url, if_modified_since=mod_date)
self.assertEqual(304, response.code)
def clear(self):
self._headers = httputil.HTTPHeaders({
'Server':
self.config.web.get('name'),
'Content-Type':
'application/json; charset=UTF-8',
'Date':
httputil.format_timestamp(time.time()),
})
self.set_default_headers()
self._write_buffer = []
self._status_code = HTTPStatus.OK.value
self._reason = HTTPStatus.OK.value
def redirect_after(self, request: httputil.HTTPServerRequest) -> Response:
"Perform a redirect to ``target``"
params = request_params(request)
date = params.get("date")
if date:
retry_after = str(
httputil.format_timestamp(
datetime.utcfromtimestamp(float(date))))
else:
retry_after = "1"
target = params.get("target", "/")
headers = [("Location", target), ("Retry-After", retry_after)]
return Response(status="303 See Other", headers=headers)
def get(self, distro=None, gpg=None):
db = self.settings['db']
(expired, dt) = yield self._cache_expired('distros', {'distro': distro})
if not expired:
self.set_status(304)
return
self.add_header('Last-Modified', httputil.format_timestamp(dt))
self.set_header('Content-Type', 'application/octet-stream')
doc = yield db.cacus.distros.find_one({'distro': distro})
if gpg:
self.write(doc['release_gpg'])
else:
self.write(doc['release_file'])
def clear(self):
"""Resets all headers and content for this response."""
self._headers = httputil.HTTPHeaders({
"Server":
"Durotar/%s" % durotar.version,
"Content-Type":
"text/html; charset=UTF-8",
"Date":
httputil.format_timestamp(time.time()),
})
self.set_default_headers()
self._write_buffer = []
self._status_code = 200
self._reason = httputil.responses[200]
def _convert_header_value(self, value):
if isinstance(value, bytes):
pass
elif isinstance(value, unicode_type):
value = value.encode("utf-8")
elif isinstance(value, numbers.Integral):
return str(value)
elif isinstance(value, datetime.datetime):
return httputil.format_timestamp(value)
else:
raise TypeError("Unsupported header value %r" % value)
if len(value) > 4000 or RequestHandler._INVALID_HEADER_CHAR_RE.search(value):
raise ValueError("Unsafe header value %r", value)
return value
def _convert_header_value(self, value):
if isinstance(value, bytes):
pass
elif isinstance(value, unicode_type):
value = value.encode('utf-8')
elif isinstance(value, numbers.Integral):
return str(value)
elif isinstance(value, datetime.datetime):
return httputil.format_timestamp(value)
else:
raise TypeError("Unsupported header value %r" % value)
if (len(value) > 4000
or RequestHandler._INVALID_HEADER_CHAR_RE.search(value)):
raise ValueError("Unsafe header value %r", value)
return value
def upload_file(self, file_data, file_path, content_type=None):
"""上传文件到oss服务器上
:param file_data: 文件的数据
:param file_path: 保存到OSS的路径
:return:
"""
oss = OssAPI(self.regional_node, self.id, self.key)
expires = format_timestamp(datetime.datetime.today() + datetime.timedelta(days=+90))
header = {'expires': expires,
'Cache-Control': 'max-age=%s' % (90*24*60*60)}
if content_type:
res = oss.put_object_from_string(self.bucket, file_path, file_data, headers=header, content_type=content_type)
else:
res = oss.put_object_from_string(self.bucket, file_path, file_data)
if 200 == res.status:
return True, file_path
else:
# log
res_message = "OSS ERROR\n%s\n%s" % (res.status, res.read())
logging.info(res_message)
return False, u'上传文件出错！'
def force_clear_cookie(self, name, path="/", domain=None):
"""Deletes the cookie with the given name.
Tornado's cookie handling currently (Jan 2018) stores cookies in a dict
keyed by name, so it can only modify one cookie with a given name per
response. The browser can store multiple cookies with the same name
but different domains and/or paths. This method lets us clear multiple
cookies with the same name.
Due to limitations of the cookie protocol, you must pass the same
path and domain to clear a cookie as were used when that cookie
was set (but there is no way to find out on the server side
which values were used for a given cookie).
"""
name = escape.native_str(name)
expires = datetime.datetime.utcnow() - datetime.timedelta(days=365)
morsel = Morsel()
morsel.set(name, '', '""')
morsel['expires'] = httputil.format_timestamp(expires)
morsel['path'] = path
if domain:
morsel['domain'] = domain
self.add_header("Set-Cookie", morsel.OutputString())
def force_clear_cookie(self, name, path="/", domain=None):
"""Deletes the cookie with the given name.
Tornado's cookie handling currently (Jan 2018) stores cookies in a dict
keyed by name, so it can only modify one cookie with a given name per
response. The browser can store multiple cookies with the same name
but different domains and/or paths. This method lets us clear multiple
cookies with the same name.
Due to limitations of the cookie protocol, you must pass the same
path and domain to clear a cookie as were used when that cookie
was set (but there is no way to find out on the server side
which values were used for a given cookie).
"""
name = escape.native_str(name)
expires = datetime.datetime.utcnow() - datetime.timedelta(days=365)
morsel = Morsel()
morsel.set(name, '', '""')
morsel['expires'] = httputil.format_timestamp(expires)
morsel['path'] = path
if domain:
morsel['domain'] = domain
self.add_header("Set-Cookie", morsel.OutputString())
def set_default_headers(self):
self._headers = httputil.HTTPHeaders(
{"Date": httputil.format_timestamp(time.time())})
def test_format(self):
format = "%A, %d-%b-%y %H:%M:%S GMT"
expected = 'Sunday, 27-Jan-13 18:43:20 GMT'
self.assertEqual(format_timestamp(self.TIMESTAMP, format),
expected)
def __init__(self,
url,
method="GET",
headers=None,
body=None,
auth_username=None,
auth_password=None,
auth_mode=None,
connect_timeout=None,
request_timeout=None,
if_modified_since=None,
follow_redirects=None,
max_redirects=None,
user_agent=None,
use_gzip=None,
network_interface=None,
streaming_callback=None,
header_callback=None,
prepare_curl_callback=None,
proxy_host=None,
proxy_port=None,
proxy_username=None,
proxy_password=None,
proxy_auth_mode=None,
allow_nonstandard_methods=None,
validate_cert=None,
ca_certs=None,
allow_ipv6=None,
client_key=None,
client_cert=None,
body_producer=None,
expect_100_continue=False,
decompress_response=None,
ssl_options=None,
key=None):
self.headers = headers
if if_modified_since:
self.headers["If-Modified-Since"] = httputil.format_timestamp(
if_modified_since)
self.proxy_host = proxy_host
self.proxy_port = proxy_port
self.proxy_username = proxy_username
self.proxy_password = proxy_password
self.proxy_auth_mode = proxy_auth_mode
self.url = url
self.method = method
self.body = body
self.body_producer = body_producer
self.auth_username = auth_username
self.auth_password = auth_password
self.auth_mode = auth_mode
self.connect_timeout = connect_timeout
self.request_timeout = request_timeout
self.follow_redirects = follow_redirects
self.max_redirects = max_redirects
self.user_agent = user_agent
self.key = key
if decompress_response is not None:
self.decompress_response = decompress_response
else:
self.decompress_response = use_gzip
self.network_interface = network_interface
self.streaming_callback = streaming_callback
self.header_callback = header_callback
self.prepare_curl_callback = prepare_curl_callback
self.allow_nonstandard_methods = allow_nonstandard_methods
self.validate_cert = validate_cert
self.ca_certs = ca_certs
self.allow_ipv6 = allow_ipv6
self.client_key = client_key
self.client_cert = client_cert
self.ssl_options = ssl_options
self.expect_100_continue = expect_100_continue
self.start_time = time.time()
def __init__(self,
url,
method="GET",
headers=None,
body=None,
auth_username=None,
auth_password=None,
auth_mode=None,
connect_timeout=None,
request_timeout=None,
if_modified_since=None,
follow_redirects=None,
max_redirects=None,
user_agent=None,
use_gzip=None,
network_interface=None,
streaming_callback=None,
header_callback=None,
prepare_curl_callback=None,
proxy_host=None,
proxy_port=None,
proxy_username=None,
proxy_password=None,
allow_nonstandard_methods=None,
validate_cert=None,
ca_certs=None,
allow_ipv6=None,
client_key=None,
client_cert=None,
parent_trace=None,
endpoint=None):
r"""All parameters except ``url`` are optional.
:arg string url: URL to fetch
:arg string method: HTTP method, e.g. "GET" or "POST"
:arg headers: Additional HTTP headers to pass on the request
:arg body: HTTP body to pass on the request
:type headers: `~tornado.httputil.HTTPHeaders` or `dict`
:arg string auth_username: Username for HTTP authentication
:arg string auth_password: Password for HTTP authentication
:arg string auth_mode: Authentication mode; default is "basic".
Allowed values are implementation-defined; ``curl_httpclient``
supports "basic" and "digest"; ``simple_httpclient`` only supports
"basic"
:arg float connect_timeout: Timeout for initial connection in seconds
:arg float request_timeout: Timeout for entire request in seconds
:arg if_modified_since: Timestamp for ``If-Modified-Since`` header
:type if_modified_since: `datetime` or `float`
:arg bool follow_redirects: Should redirects be followed automatically
or return the 3xx response?
:arg int max_redirects: Limit for ``follow_redirects``
:arg string user_agent: String to send as ``User-Agent`` header
:arg bool use_gzip: Request gzip encoding from the server
:arg string network_interface: Network interface to use for request.
``curl_httpclient`` only; see note below.
:arg callable streaming_callback: If set, ``streaming_callback`` will
be run with each chunk of data as it is received, and
``HTTPResponse.body`` and ``HTTPResponse.buffer`` will be empty in
the final response.
:arg callable header_callback: If set, ``header_callback`` will
be run with each header line as it is received (including the
first line, e.g. ``HTTP/1.0 200 OK\r\n``, and a final line
containing only ``\r\n``. All lines include the trailing newline
characters). ``HTTPResponse.headers`` will be empty in the final
response. This is most useful in conjunction with
``streaming_callback``, because it's the only way to get access to
header data while the request is in progress.
:arg callable prepare_curl_callback: If set, will be called with
a ``pycurl.Curl`` object to allow the application to make additional
``setopt`` calls.
:arg string proxy_host: HTTP proxy hostname. To use proxies,
``proxy_host`` and ``proxy_port`` must be set; ``proxy_username`` and
``proxy_pass`` are optional. Proxies are currently only supported
with ``curl_httpclient``.
:arg int proxy_port: HTTP proxy port
:arg string proxy_username: HTTP proxy username
:arg string proxy_password: HTTP proxy password
:arg bool allow_nonstandard_methods: Allow unknown values for ``method``
argument?
:arg bool validate_cert: For HTTPS requests, validate the server's
certificate?
:arg string ca_certs: filename of CA certificates in PEM format,
or None to use defaults. See note below when used with
``curl_httpclient``.
:arg bool allow_ipv6: Use IPv6 when available? Default is false in
``simple_httpclient`` and true in ``curl_httpclient``
:arg string client_key: Filename for client SSL key, if any. See
note below when used with ``curl_httpclient``.
:arg string client_cert: Filename for client SSL certificate, if any.
See note below when used with ``curl_httpclient``.
:arg string parent_trace: parent trace id.
:arg string endpoint: request endpoint.
.. note::
When using ``curl_httpclient`` certain options may be
inherited by subsequent fetches because ``pycurl`` does
not allow them to be cleanly reset. This applies to the
``ca_certs``, ``client_key``, ``client_cert``, and
``network_interface`` arguments. If you use these
options, you should pass them on every request (you don't
have to always use the same values, but it's not possible
to mix requests that specify these options with ones that
use the defaults).
.. versionadded:: 3.1
The ``auth_mode`` argument.
"""
# Note that some of these attributes go through property setters
# defined below.
self.headers = headers
if if_modified_since:
self.headers["If-Modified-Since"] = httputil.format_timestamp(
if_modified_since)
self.proxy_host = proxy_host
self.proxy_port = proxy_port
self.proxy_username = proxy_username
self.proxy_password = proxy_password
self.url = url
self.method = method
self.body = body
self.auth_username = auth_username
self.auth_password = auth_password
self.auth_mode = auth_mode
self.connect_timeout = connect_timeout
self.request_timeout = request_timeout
self.follow_redirects = follow_redirects
self.max_redirects = max_redirects
self.user_agent = user_agent
self.use_gzip = use_gzip
self.network_interface = network_interface
self.streaming_callback = streaming_callback
self.header_callback = header_callback
self.prepare_curl_callback = prepare_curl_callback
self.allow_nonstandard_methods = allow_nonstandard_methods
self.validate_cert = validate_cert
self.ca_certs = ca_certs
self.allow_ipv6 = allow_ipv6
self.client_key = client_key
self.client_cert = client_cert
self.start_time = time.time()
self._parent_trace = parent_trace
self._endpoint = endpoint
self.trace = None
if options.client_trace:
if self._parent_trace is None:
self.trace = Trace(method)
else:
self.trace = self._parent_trace.child(method)
if self._endpoint is not None:
self.trace.set_endpoint(self._endpoint)
else:
self._endpoint = Endpoint(ipv4=socket.gethostbyname(
socket.gethostname()),
port=0,
service_name=service_name)
self.trace.set_endpoint(self._endpoint)
self.headers.set('X-B3-TraceId', [hex_str(self.trace.trace_id)])
self.headers.set('X-B3-SpanId', [hex_str(self.trace.span_id)])
if trace.parent_span_id is not None:
self.headers.set('X-B3-ParentSpanId',
[hex_str(self.trace.parent_span_id)])
self.trace.record(Annotation.string('HTTPClient REQ', self.url))
def set_cookie(self,
name,
value,
domain=None,
expires=None,
path="/",
expires_days=None,
**kwargs):
"""Sets an outgoing cookie name/value with the given options.
Newly-set cookies are not immediately visible via `get_cookie`;
they are not present until the next request.
expires may be a numeric timestamp as returned by `time.time`,
a time tuple as returned by `time.gmtime`, or a
`datetime.datetime` object.
Additional keyword arguments are set on the cookies.Morsel
directly.
https://docs.python.org/3/library/http.cookies.html#http.cookies.Morsel
---
Taken from Tornado's web module:
https://github.com/tornadoweb/tornado/blob/
627eafb3ce21a777981c37a5867b5f1956a4dc16/tornado/web.py#L528
The main reason for bundling this in here is to allow use of the
SameSite attribute for cookies via our vendored cookies library.
"""
# The cookie library only accepts type str, in both python 2 and 3
name = native_str(name)
value = native_str(value)
if re.search(r"[\x00-\x20]", name + value):
# Don't let us accidentally inject bad stuff
raise ValueError("Invalid cookie %r: %r" % (name, value))
if not hasattr(self, "_new_cookie"):
self._new_cookie = cookies.SimpleCookie()
if name in self._new_cookie:
del self._new_cookie[name]
self._new_cookie[name] = value
morsel = self._new_cookie[name]
if domain:
morsel["domain"] = domain
if expires_days is not None and not expires:
expires = datetime.utcnow() + timedelta(days=expires_days)
if expires:
morsel["expires"] = httputil.format_timestamp(expires)
if path:
morsel["path"] = path
for k, v in kwargs.items():
if k == 'max_age':
k = 'max-age'
# skip falsy values for httponly and secure flags because
# SimpleCookie sets them regardless
if k in ['httponly', 'secure'] and not v:
continue
morsel[k] = v
def __init__(self, url, method="GET", headers=None, body=None,
auth_username=None, auth_password=None,
connect_timeout=None, request_timeout=None,
if_modified_since=None, follow_redirects=None,
max_redirects=None, user_agent=None, use_gzip=None,
network_interface=None, streaming_callback=None,
header_callback=None, prepare_curl_callback=None,
proxy_host=None, proxy_port=None, proxy_username=None,
proxy_password=None, allow_nonstandard_methods=None,
validate_cert=None, ca_certs=None,
allow_ipv6=None,
client_key=None, client_cert=None):
r"""All parameters except ``url`` are optional.
:arg string url: URL to fetch
:arg string method: HTTP method, e.g. "GET" or "POST"
:arg headers: Additional HTTP headers to pass on the request
:type headers: `~tornado.httputil.HTTPHeaders` or `dict`
:arg string auth_username: Username for HTTP "Basic" authentication
:arg string auth_password: Password for HTTP "Basic" authentication
:arg float connect_timeout: Timeout for initial connection in seconds
:arg float request_timeout: Timeout for entire request in seconds
:arg if_modified_since: Timestamp for ``If-Modified-Since`` header
:type if_modified_since: `datetime` or `float`
:arg bool follow_redirects: Should redirects be followed automatically
or return the 3xx response?
:arg int max_redirects: Limit for ``follow_redirects``
:arg string user_agent: String to send as ``User-Agent`` header
:arg bool use_gzip: Request gzip encoding from the server
:arg string network_interface: Network interface to use for request
:arg callable streaming_callback: If set, ``streaming_callback`` will
be run with each chunk of data as it is received, and
``HTTPResponse.body`` and ``HTTPResponse.buffer`` will be empty in
the final response.
:arg callable header_callback: If set, ``header_callback`` will
be run with each header line as it is received (including the
first line, e.g. ``HTTP/1.0 200 OK\r\n``, and a final line
containing only ``\r\n``. All lines include the trailing newline
characters). ``HTTPResponse.headers`` will be empty in the final
response. This is most useful in conjunction with
``streaming_callback``, because it's the only way to get access to
header data while the request is in progress.
:arg callable prepare_curl_callback: If set, will be called with
a ``pycurl.Curl`` object to allow the application to make additional
``setopt`` calls.
:arg string proxy_host: HTTP proxy hostname. To use proxies,
``proxy_host`` and ``proxy_port`` must be set; ``proxy_username`` and
``proxy_pass`` are optional. Proxies are currently only supported
with ``curl_httpclient``.
:arg int proxy_port: HTTP proxy port
:arg string proxy_username: HTTP proxy username
:arg string proxy_password: HTTP proxy password
:arg bool allow_nonstandard_methods: Allow unknown values for ``method``
argument?
:arg bool validate_cert: For HTTPS requests, validate the server's
certificate?
:arg string ca_certs: filename of CA certificates in PEM format,
or None to use defaults. Note that in ``curl_httpclient``, if
any request uses a custom ``ca_certs`` file, they all must (they
don't have to all use the same ``ca_certs``, but it's not possible
to mix requests with ``ca_certs`` and requests that use the defaults.
:arg bool allow_ipv6: Use IPv6 when available? Default is false in
``simple_httpclient`` and true in ``curl_httpclient``
:arg string client_key: Filename for client SSL key, if any
:arg string client_cert: Filename for client SSL certificate, if any
"""
if headers is None:
headers = httputil.HTTPHeaders()
if if_modified_since:
headers["If-Modified-Since"] = httputil.format_timestamp(
if_modified_since)
self.proxy_host = proxy_host
self.proxy_port = proxy_port
self.proxy_username = proxy_username
self.proxy_password = proxy_password
self.url = url
self.method = method
self.headers = headers
self.body = utf8(body)
self.auth_username = auth_username
self.auth_password = auth_password
self.connect_timeout = connect_timeout
self.request_timeout = request_timeout
self.follow_redirects = follow_redirects
self.max_redirects = max_redirects
self.user_agent = user_agent
self.use_gzip = use_gzip
self.network_interface = network_interface
self.streaming_callback = stack_context.wrap(streaming_callback)
self.header_callback = stack_context.wrap(header_callback)
self.prepare_curl_callback = stack_context.wrap(prepare_curl_callback)
self.allow_nonstandard_methods = allow_nonstandard_methods
self.validate_cert = validate_cert
self.ca_certs = ca_certs
self.allow_ipv6 = allow_ipv6
self.client_key = client_key
self.client_cert = client_cert
self.start_time = time.time()
def check(self, value):
self.assertEqual(format_timestamp(value), self.EXPECTED)
def __init__(
self,
url: str,
method: str = "GET",
headers: Union[Dict[str, str], httputil.HTTPHeaders] = None,
body: Union[bytes, str] = None,
auth_username: str = None,
auth_password: str = None,
auth_mode: str = None,
connect_timeout: float = None,
request_timeout: float = None,
if_modified_since: Union[float, datetime.datetime] = None,
follow_redirects: bool = None,
max_redirects: int = None,
user_agent: str = None,
use_gzip: bool = None,
network_interface: str = None,
streaming_callback: Callable[[bytes], None] = None,
header_callback: Callable[[str], None] = None,
prepare_curl_callback: Callable[[Any], None] = None,
proxy_host: str = None,
proxy_port: int = None,
proxy_username: str = None,
proxy_password: str = None,
proxy_auth_mode: str = None,
allow_nonstandard_methods: bool = None,
validate_cert: bool = None,
ca_certs: str = None,
allow_ipv6: bool = None,
client_key: str = None,
client_cert: str = None,
body_producer: Callable[[Callable[[bytes], None]],
"Future[None]"] = None,
expect_100_continue: bool = False,
decompress_response: bool = None,
ssl_options: Union[Dict[str, Any], ssl.SSLContext] = None,
) -> None:
# Note that some of these attributes go through property setters
# defined below.
self.headers = headers
if if_modified_since:
self.headers["If-Modified-Since"] = httputil.format_timestamp(
if_modified_since)
self.proxy_host = proxy_host
self.proxy_port = proxy_port
self.proxy_username = proxy_username
self.proxy_password = proxy_password
self.proxy_auth_mode = proxy_auth_mode
self.url = url
self.method = method
self.body = body
self.body_producer = body_producer
self.auth_username = auth_username
self.auth_password = auth_password
self.auth_mode = auth_mode
self.connect_timeout = connect_timeout
self.request_timeout = request_timeout
self.follow_redirects = follow_redirects
self.max_redirects = max_redirects
self.user_agent = user_agent
if decompress_response is not None:
self.decompress_response = decompress_response # type: Optional[bool]
else:
self.decompress_response = use_gzip
self.network_interface = network_interface
self.streaming_callback = streaming_callback
self.header_callback = header_callback
self.prepare_curl_callback = prepare_curl_callback
self.allow_nonstandard_methods = allow_nonstandard_methods
self.validate_cert = validate_cert
self.ca_certs = ca_certs
self.allow_ipv6 = allow_ipv6
self.client_key = client_key
self.client_cert = client_cert
self.ssl_options = ssl_options
self.expect_100_continue = expect_100_continue
self.start_time = time.time()
def get(self): # def get(self, course_code, assignment_code=None):
[course_code, assignment_code] = self.get_params(["course_id", "assignment_id"])
if not (course_code and assignment_code):
note = "Assigment call requires both a course code and an assignment code!!"
self.log.info(note)
self.finish({"success": False, "note": note})
return
this_user = self.nbex_user
if not course_code in this_user["courses"]:
note = f"User not subscribed to course {course_code}"
self.log.info(note)
self.finish({"success": False, "note": note})
return
# Find the course being referred to
with scoped_session() as session:
course = Course.find_by_code(
db=session, code=course_code, org_id=this_user["org_id"], log=self.log
)
if course is None:
note = f"Course {course_code} does not exist"
self.log.info(note)
self.finish({"success": False, "note": note})
return # needs a proper 'fail' here
note = ""
self.log.debug(f"Course:{course_code} assignment:{assignment_code}")
# The location for the data-object is actually held in the 'released' action for the given assignment
# We want the last one...
assignment = AssignmentModel.find_by_code(
db=session,
code=assignment_code,
course_id=course.id,
action=AssignmentActions.released.value,
)
if assignment is None:
note = f"Assignment {assignment_code} does not exist"
self.log.info(note)
self.finish({"success": False, "note": note})
return # needs a proper 'fail' here
self._headers = httputil.HTTPHeaders(
{
"Content-Type": "application/gzip",
"Date": httputil.format_timestamp(time.time()),
}
)
data = b""
release_file = None
action = Action.find_most_recent_action(
db=session,
assignment_id=assignment.id,
action=AssignmentActions.released,
log=self.log,
)
release_file = action.location
if release_file:
try:
with open(release_file, "r+b") as handle:
data = handle.read()
except Exception as e: # TODO: exception handling
self.log.warning(f"Error: {e}") # TODO: improve error message
self.log.info(f"Unable to open file")
# error 500??
raise Exception
self.log.info(
f"Adding action {AssignmentActions.fetched.value} for user {this_user['id']} against assignment {assignment.id}"
)
action = Action(
user_id=this_user["id"],
assignment_id=assignment.id,
action=AssignmentActions.fetched,
location=release_file,
)
session.add(action)
self.log.info("record of fetch action committed")
self.finish(data)
else:
self.log.info("no release file found")
raise Exception
def __init__(self,
url,
method="GET",
headers=None,
body=None,
auth_username=None,
auth_password=None,
auth_mode=None,
connect_timeout=None,
request_timeout=None,
if_modified_since=None,
follow_redirects=None,
max_redirects=None,
user_agent=None,
use_gzip=None,
network_interface=None,
streaming_callback=None,
header_callback=None,
prepare_curl_callback=None,
proxy_host=None,
proxy_port=None,
proxy_username=None,
proxy_password=None,
allow_nonstandard_methods=None,
validate_cert=None,
ca_certs=None,
allow_ipv6=None,
client_key=None,
client_cert=None):
r"""All parameters except ``url`` are optional.
:arg string url: URL to fetch
:arg string method: HTTP method, e.g. "GET" or "POST"
:arg headers: Additional HTTP headers to pass on the request
:type headers: `~tornado.httputil.HTTPHeaders` or `dict`
:arg string auth_username: Username for HTTP authentication
:arg string auth_password: Password for HTTP authentication
:arg string auth_mode: Authentication mode; default is "basic".
Allowed values are implementation-defined; ``curl_httpclient``
supports "basic" and "digest"; ``simple_httpclient`` only supports
"basic"
:arg float connect_timeout: Timeout for initial connection in seconds
:arg float request_timeout: Timeout for entire request in seconds
:arg if_modified_since: Timestamp for ``If-Modified-Since`` header
:type if_modified_since: `datetime` or `float`
:arg bool follow_redirects: Should redirects be followed automatically
or return the 3xx response?
:arg int max_redirects: Limit for ``follow_redirects``
:arg string user_agent: String to send as ``User-Agent`` header
:arg bool use_gzip: Request gzip encoding from the server
:arg string network_interface: Network interface to use for request
:arg callable streaming_callback: If set, ``streaming_callback`` will
be run with each chunk of data as it is received, and
``HTTPResponse.body`` and ``HTTPResponse.buffer`` will be empty in
the final response.
:arg callable header_callback: If set, ``header_callback`` will
be run with each header line as it is received (including the
first line, e.g. ``HTTP/1.0 200 OK\r\n``, and a final line
containing only ``\r\n``. All lines include the trailing newline
characters). ``HTTPResponse.headers`` will be empty in the final
response. This is most useful in conjunction with
``streaming_callback``, because it's the only way to get access to
header data while the request is in progress.
:arg callable prepare_curl_callback: If set, will be called with
a ``pycurl.Curl`` object to allow the application to make additional
``setopt`` calls.
:arg string proxy_host: HTTP proxy hostname. To use proxies,
``proxy_host`` and ``proxy_port`` must be set; ``proxy_username`` and
``proxy_pass`` are optional. Proxies are currently only supported
with ``curl_httpclient``.
:arg int proxy_port: HTTP proxy port
:arg string proxy_username: HTTP proxy username
:arg string proxy_password: HTTP proxy password
:arg bool allow_nonstandard_methods: Allow unknown values for ``method``
argument?
:arg bool validate_cert: For HTTPS requests, validate the server's
certificate?
:arg string ca_certs: filename of CA certificates in PEM format,
or None to use defaults. Note that in ``curl_httpclient``, if
any request uses a custom ``ca_certs`` file, they all must (they
don't have to all use the same ``ca_certs``, but it's not possible
to mix requests with ``ca_certs`` and requests that use the defaults.
:arg bool allow_ipv6: Use IPv6 when available? Default is false in
``simple_httpclient`` and true in ``curl_httpclient``
:arg string client_key: Filename for client SSL key, if any
:arg string client_cert: Filename for client SSL certificate, if any
"""
if headers is None:
headers = httputil.HTTPHeaders()
if if_modified_since:
headers["If-Modified-Since"] = httputil.format_timestamp(
if_modified_since)
self.proxy_host = proxy_host
self.proxy_port = proxy_port
self.proxy_username = proxy_username
self.proxy_password = proxy_password
self.url = url
self.method = method
self.headers = headers
self.body = utf8(body)
self.auth_username = auth_username
self.auth_password = auth_password
self.auth_mode = auth_mode
self.connect_timeout = connect_timeout
self.request_timeout = request_timeout
self.follow_redirects = follow_redirects
self.max_redirects = max_redirects
self.user_agent = user_agent
self.use_gzip = use_gzip
self.network_interface = network_interface
self.streaming_callback = stack_context.wrap(streaming_callback)
self.header_callback = stack_context.wrap(header_callback)
self.prepare_curl_callback = stack_context.wrap(prepare_curl_callback)
self.allow_nonstandard_methods = allow_nonstandard_methods
self.validate_cert = validate_cert
self.ca_certs = ca_certs
self.allow_ipv6 = allow_ipv6
self.client_key = client_key
self.client_cert = client_cert
self.start_time = time.time()
def test_if_modified_since(self):
http_date = datetime.datetime.utcnow()
request = HTTPRequest('http://example.com', if_modified_since=http_date)
self.assertEqual(request.headers,
{'If-Modified-Since': format_timestamp(http_date)})
def set_secure_cookie(self, key, value, expires_days=30):
if self.request is not None:
expires = datetime.datetime.utcnow() + datetime.timedelta(days=expires_days)
self.request.addCookie(key, value, expires=format_timestamp(expires), path="/")
def check(self, value):
self.assertEqual(format_timestamp(value), self.EXPECTED)
def set_default_headers(self):
self._headers = httputil.HTTPHeaders({
"Date": httputil.format_timestamp(time.time())
})
def __init__(self, url, method="GET", headers=None, body=None,
auth_username=None, auth_password=None, auth_mode=None,
connect_timeout=None, request_timeout=None,
if_modified_since=None, follow_redirects=None,
max_redirects=None, user_agent=None, use_gzip=None,
network_interface=None, streaming_callback=None,
header_callback=None, prepare_curl_callback=None,
proxy_host=None, proxy_port=None, proxy_username=None,
proxy_password=None, allow_nonstandard_methods=None,
validate_cert=None, ca_certs=None,
allow_ipv6=None,
client_key=None, client_cert=None, body_producer=None,
expect_100_continue=False):
r"""All parameters except ``url`` are optional.
:arg string url: URL to fetch
:arg string method: HTTP method, e.g. "GET" or "POST"
:arg headers: Additional HTTP headers to pass on the request
:type headers: `~tornado.httputil.HTTPHeaders` or `dict`
:arg body: HTTP request body as a string (byte or unicode; if unicode
the utf-8 encoding will be used)
:arg body_producer: Callable used for lazy/asynchronous request bodies.
It is called with one argument, a ``write`` function, and should
return a `.Future`. It should call the write function with new
data as it becomes available. The write function returns a
`.Future` which can be used for flow control.
Only one of ``body`` and ``body_producer`` may
be specified. ``body_producer`` is not supported on
``curl_httpclient``. When using ``body_producer`` it is recommended
to pass a ``Content-Length`` in the headers as otherwise chunked
encoding will be used, and many servers do not support chunked
encoding on requests. New in Tornado 4.0
:arg string auth_username: Username for HTTP authentication
:arg string auth_password: Password for HTTP authentication
:arg string auth_mode: Authentication mode; default is "basic".
Allowed values are implementation-defined; ``curl_httpclient``
supports "basic" and "digest"; ``simple_httpclient`` only supports
"basic"
:arg float connect_timeout: Timeout for initial connection in seconds
:arg float request_timeout: Timeout for entire request in seconds
:arg if_modified_since: Timestamp for ``If-Modified-Since`` header
:type if_modified_since: `datetime` or `float`
:arg bool follow_redirects: Should redirects be followed automatically
or return the 3xx response?
:arg int max_redirects: Limit for ``follow_redirects``
:arg string user_agent: String to send as ``User-Agent`` header
:arg bool use_gzip: Request gzip encoding from the server
:arg string network_interface: Network interface to use for request.
``curl_httpclient`` only; see note below.
:arg callable streaming_callback: If set, ``streaming_callback`` will
be run with each chunk of data as it is received, and
``HTTPResponse.body`` and ``HTTPResponse.buffer`` will be empty in
the final response.
:arg callable header_callback: If set, ``header_callback`` will
be run with each header line as it is received (including the
first line, e.g. ``HTTP/1.0 200 OK\r\n``, and a final line
containing only ``\r\n``. All lines include the trailing newline
characters). ``HTTPResponse.headers`` will be empty in the final
response. This is most useful in conjunction with
``streaming_callback``, because it's the only way to get access to
header data while the request is in progress.
:arg callable prepare_curl_callback: If set, will be called with
a ``pycurl.Curl`` object to allow the application to make additional
``setopt`` calls.
:arg string proxy_host: HTTP proxy hostname. To use proxies,
``proxy_host`` and ``proxy_port`` must be set; ``proxy_username`` and
``proxy_pass`` are optional. Proxies are currently only supported
with ``curl_httpclient``.
:arg int proxy_port: HTTP proxy port
:arg string proxy_username: HTTP proxy username
:arg string proxy_password: HTTP proxy password
:arg bool allow_nonstandard_methods: Allow unknown values for ``method``
argument?
:arg bool validate_cert: For HTTPS requests, validate the server's
certificate?
:arg string ca_certs: filename of CA certificates in PEM format,
or None to use defaults. See note below when used with
``curl_httpclient``.
:arg bool allow_ipv6: Use IPv6 when available? Default is false in
``simple_httpclient`` and true in ``curl_httpclient``
:arg string client_key: Filename for client SSL key, if any. See
note below when used with ``curl_httpclient``.
:arg string client_cert: Filename for client SSL certificate, if any.
See note below when used with ``curl_httpclient``.
:arg bool expect_100_continue: If true, send the
``Expect: 100-continue`` header and wait for a continue response
before sending the request body. Only supported with
simple_httpclient.
.. note::
When using ``curl_httpclient`` certain options may be
inherited by subsequent fetches because ``pycurl`` does
not allow them to be cleanly reset. This applies to the
``ca_certs``, ``client_key``, ``client_cert``, and
``network_interface`` arguments. If you use these
options, you should pass them on every request (you don't
have to always use the same values, but it's not possible
to mix requests that specify these options with ones that
use the defaults).
.. versionadded:: 3.1
The ``auth_mode`` argument.
.. versionadded:: 4.0
The ``body_producer`` and ``expect_100_continue`` arguments.
"""
# Note that some of these attributes go through property setters
# defined below.
self.headers = headers
if if_modified_since:
self.headers["If-Modified-Since"] = httputil.format_timestamp(
if_modified_since)
self.proxy_host = proxy_host
self.proxy_port = proxy_port
self.proxy_username = proxy_username
self.proxy_password = proxy_password
self.url = url
self.method = method
self.body = body
self.body_producer = body_producer
self.auth_username = auth_username
self.auth_password = auth_password
self.auth_mode = auth_mode
self.connect_timeout = connect_timeout
self.request_timeout = request_timeout
self.follow_redirects = follow_redirects
self.max_redirects = max_redirects
self.user_agent = user_agent
self.use_gzip = use_gzip
self.network_interface = network_interface
self.streaming_callback = streaming_callback
self.header_callback = header_callback
self.prepare_curl_callback = prepare_curl_callback
self.allow_nonstandard_methods = allow_nonstandard_methods
self.validate_cert = validate_cert
self.ca_certs = ca_certs
self.allow_ipv6 = allow_ipv6
self.client_key = client_key
self.client_cert = client_cert
self.expect_100_continue = expect_100_continue
self.start_time = time.time()
def fetch(self, target, refresh=False, cache=True, delay=None,
follow=True, extract=None, **kwargs):
"""Fetch a URL from the wild, but first check the Cache.
Args:
target (str or HTTPRequest): to be fetched.
refresh (bool, optional): should the CacheClient ask the remote
source to refresh cached files? Defaults to False.
cache (bool, optional): should results be cached? Defaults to True.
delay (int, optional): a period, in seconds, for which the client
should delay before sending the next request after a successful
fetch.
follow (bool, optional): should redirects be followed? If False,
the Response object will only contain a string to the redirect
url target. Defaults to True.
extract (str, optional): if supplied, the Client will try to
extract a filename of `extract` from any resulting compressed
file.
**kwargs (misc., optional): any additional keyword arguments that
should be passed when a new HTTPRequest is initialized.
Returns (/ Raises):
response (cache.Response or None): a named tuple containing values:
- `url` (string): the url of the fetch/cache load.
- `buffer` (BytesIO): the body of the fetch result.
- `fresh` (bool): True if the Response object is the result of
a fresh response from the target server.
or None if an error occurred (which is logged).
"""
request = self._cached_http_request(target, follow_redirects=follow,
**kwargs)
self._log.debug("Fetching file @ {}".format(request.url))
if not refresh and IF_MODIFIED_SINCE in request.headers:
self._log.debug("Have cached file, not asking for a refresh.")
response = self.cache.load(request.url)
raise gen.Return(response)
elif IF_MODIFIED_SINCE in request.headers:
last_mod = request.headers[IF_MODIFIED_SINCE]
age = datetime.datetime.now() - last_mod
if age.seconds < REFRESH_COOLDOWN:
self._log.debug("Have recent cached file, not refreshing.")
raise gen.Return(self.cache.load(request.url))
else:
request.headers[IF_MODIFIED_SINCE] = format_timestamp(last_mod)
try:
response = yield self._client.fetch(request)
except HTTPError as err:
if err.code == FILE_UNCHANGED:
self._log.debug("File unchanged, using cached version.")
raise gen.Return(self.cache.load(request.url))
# If we get a 302, and we're expecting it, return the location and
# fresh to indicate that the destination is a new one (since we
# had to reach out to the server.
elif err.code == SOFT_REDIRECT and not follow:
loc = err.response.headers[LOCATION_HEADER]
self._log.debug('Redirected to {}, not following'.format(loc))
response = Response(BytesIO(loc), request.url, True)
raise gen.Return(response)
else:
self._log.error(
"{0} ({1}) fetching {2}".format(err, err.code, request.url))
raise gen.Return(None)
except Exception as excp:
self._log.exception(excp)
raise gen.Return(None)
else:
self._log.debug("Got fresh file @ {0}".format(request.url))
if extract:
response.buffer = decompress_response(response.buffer, extract)
if cache:
self._log.debug("Caching {0}".format(request.url))
self.cache_response(response, overwrite=True)
response = Response(response.buffer, request.url, True)
raise gen.Return(response)
finally:
if delay:
self._log.debug("Pausing @ {0} for {1} sec(s)".format(
self.ioloop.time(), delay))
yield gen.sleep(delay)
def test_if_modified_since(self):
http_date = datetime.datetime.utcnow()
request = HTTPRequest('http://example.com', if_modified_since=http_date)
self.assertEqual(request.headers,
{'If-Modified-Since': format_timestamp(http_date)})
def __init__(self, url, method="GET", headers=None, body=None,
auth_username=None, auth_password=None, auth_mode=None,
connect_timeout=None, request_timeout=None,
if_modified_since=None, follow_redirects=None,
max_redirects=None, user_agent=None, use_gzip=None,
network_interface=None, streaming_callback=None,
header_callback=None, prepare_curl_callback=None,
proxy_host=None, proxy_port=None, proxy_username=None,
proxy_password=None, allow_nonstandard_methods=None,
validate_cert=None, ca_certs=None,
allow_ipv6=None,
client_key=None, client_cert=None,
parent_trace=None, endpoint=None):
r"""All parameters except ``url`` are optional.
:arg string url: URL to fetch
:arg string method: HTTP method, e.g. "GET" or "POST"
:arg headers: Additional HTTP headers to pass on the request
:arg body: HTTP body to pass on the request
:type headers: `~tornado.httputil.HTTPHeaders` or `dict`
:arg string auth_username: Username for HTTP authentication
:arg string auth_password: Password for HTTP authentication
:arg string auth_mode: Authentication mode; default is "basic".
Allowed values are implementation-defined; ``curl_httpclient``
supports "basic" and "digest"; ``simple_httpclient`` only supports
"basic"
:arg float connect_timeout: Timeout for initial connection in seconds
:arg float request_timeout: Timeout for entire request in seconds
:arg if_modified_since: Timestamp for ``If-Modified-Since`` header
:type if_modified_since: `datetime` or `float`
:arg bool follow_redirects: Should redirects be followed automatically
or return the 3xx response?
:arg int max_redirects: Limit for ``follow_redirects``
:arg string user_agent: String to send as ``User-Agent`` header
:arg bool use_gzip: Request gzip encoding from the server
:arg string network_interface: Network interface to use for request.
``curl_httpclient`` only; see note below.
:arg callable streaming_callback: If set, ``streaming_callback`` will
be run with each chunk of data as it is received, and
``HTTPResponse.body`` and ``HTTPResponse.buffer`` will be empty in
the final response.
:arg callable header_callback: If set, ``header_callback`` will
be run with each header line as it is received (including the
first line, e.g. ``HTTP/1.0 200 OK\r\n``, and a final line
containing only ``\r\n``. All lines include the trailing newline
characters). ``HTTPResponse.headers`` will be empty in the final
response. This is most useful in conjunction with
``streaming_callback``, because it's the only way to get access to
header data while the request is in progress.
:arg callable prepare_curl_callback: If set, will be called with
a ``pycurl.Curl`` object to allow the application to make additional
``setopt`` calls.
:arg string proxy_host: HTTP proxy hostname. To use proxies,
``proxy_host`` and ``proxy_port`` must be set; ``proxy_username`` and
``proxy_pass`` are optional. Proxies are currently only supported
with ``curl_httpclient``.
:arg int proxy_port: HTTP proxy port
:arg string proxy_username: HTTP proxy username
:arg string proxy_password: HTTP proxy password
:arg bool allow_nonstandard_methods: Allow unknown values for ``method``
argument?
:arg bool validate_cert: For HTTPS requests, validate the server's
certificate?
:arg string ca_certs: filename of CA certificates in PEM format,
or None to use defaults. See note below when used with
``curl_httpclient``.
:arg bool allow_ipv6: Use IPv6 when available? Default is false in
``simple_httpclient`` and true in ``curl_httpclient``
:arg string client_key: Filename for client SSL key, if any. See
note below when used with ``curl_httpclient``.
:arg string client_cert: Filename for client SSL certificate, if any.
See note below when used with ``curl_httpclient``.
:arg string parent_trace: parent trace id.
:arg string endpoint: request endpoint.
.. note::
When using ``curl_httpclient`` certain options may be
inherited by subsequent fetches because ``pycurl`` does
not allow them to be cleanly reset. This applies to the
``ca_certs``, ``client_key``, ``client_cert``, and
``network_interface`` arguments. If you use these
options, you should pass them on every request (you don't
have to always use the same values, but it's not possible
to mix requests that specify these options with ones that
use the defaults).
.. versionadded:: 3.1
The ``auth_mode`` argument.
"""
# Note that some of these attributes go through property setters
# defined below.
self.headers = headers
if if_modified_since:
self.headers["If-Modified-Since"] = httputil.format_timestamp(
if_modified_since)
self.proxy_host = proxy_host
self.proxy_port = proxy_port
self.proxy_username = proxy_username
self.proxy_password = proxy_password
self.url = url
self.method = method
self.body = body
self.auth_username = auth_username
self.auth_password = auth_password
self.auth_mode = auth_mode
self.connect_timeout = connect_timeout
self.request_timeout = request_timeout
self.follow_redirects = follow_redirects
self.max_redirects = max_redirects
self.user_agent = user_agent
self.use_gzip = use_gzip
self.network_interface = network_interface
self.streaming_callback = streaming_callback
self.header_callback = header_callback
self.prepare_curl_callback = prepare_curl_callback
self.allow_nonstandard_methods = allow_nonstandard_methods
self.validate_cert = validate_cert
self.ca_certs = ca_certs
self.allow_ipv6 = allow_ipv6
self.client_key = client_key
self.client_cert = client_cert
self.start_time = time.time()
self._parent_trace = parent_trace
self._endpoint = endpoint
self.trace = None
if options.client_trace:
if self._parent_trace is None:
self.trace = Trace(method)
else:
self.trace = self._parent_trace.child(method)
if self._endpoint is not None:
self.trace.set_endpoint(self._endpoint)
else:
self._endpoint = Endpoint(ipv4=socket.gethostbyname(socket.gethostname()), port=0, service_name=service_name)
self.trace.set_endpoint(self._endpoint)
self.headers.set('X-B3-TraceId', [hex_str(self.trace.trace_id)])
self.headers.set('X-B3-SpanId', [hex_str(self.trace.span_id)])
if trace.parent_span_id is not None:
self.headers.set('X-B3-ParentSpanId', [hex_str(self.trace.parent_span_id)])
self.trace.record(Annotation.string('HTTPClient REQ', self.url))
def set_cookie(self,
name,
value,
expires_days=30,
version=None,
domain=None,
expires=None,
path="/",
**kwargs):
""" Sets the given cookie name/value with the given options. Set value
to None to clear. The cookie value is secured using
`flexx.config.cookie_secret`; don't forget to set that config
value in your server. Additional keyword arguments are set on
the Cookie.Morsel directly.
"""
# This code is taken (in modified form) from the Tornado project
# Copyright 2009 Facebook
# Licensed under the Apache License, Version 2.0
# Assume tornado is available ...
from tornado.escape import native_str
from tornado.httputil import format_timestamp
from tornado.web import create_signed_value
# Clear cookie?
if value is None:
value = ""
expires = datetime.datetime.utcnow() - datetime.timedelta(days=365)
else:
secret = config.cookie_secret
value = create_signed_value(secret,
name,
value,
version=version,
key_version=None)
# The cookie library only accepts type str, in both python 2 and 3
name = native_str(name)
value = native_str(value)
if re.search(r"[\x00-\x20]", name + value):
# Don't let us accidentally inject bad stuff
raise ValueError("Invalid cookie %r: %r" % (name, value))
if name in self._cookies:
del self._cookies[name]
self._cookies[name] = value
morsel = self._cookies[name]
if domain:
morsel["domain"] = domain
if expires_days is not None and not expires:
expires = datetime.datetime.utcnow() + datetime.timedelta(
days=expires_days)
if expires:
morsel["expires"] = format_timestamp(expires)
if path:
morsel["path"] = path
for k, v in kwargs.items():
if k == 'max_age':
k = 'max-age'
# skip falsy values for httponly and secure flags because
# SimpleCookie sets them regardless
if k in ['httponly', 'secure'] and not v:
continue
morsel[k] = v
self.send_command(
'EXEC', 'document.cookie = "%s";' %
morsel.OutputString().replace('"', '\\"'))
def __init__(self,
url,
method="GET",
headers=None,
body=None,
auth_username=None,
auth_password=None,
auth_mode=None,
connect_timeout=None,
request_timeout=None,
if_modified_since=None,
follow_redirects=None,
max_redirects=None,
user_agent=None,
use_gzip=None,
network_interface=None,
streaming_callback=None,
header_callback=None,
prepare_curl_callback=None,
proxy_host=None,
proxy_port=None,
proxy_username=None,
proxy_password=None,
allow_nonstandard_methods=None,
validate_cert=None,
ca_certs=None,
allow_ipv6=None,
client_key=None,
client_cert=None,
body_producer=None,
expect_100_continue=False,
decompress_response=None,
ssl_options=None):
r"""除了 ``url`` 以外所有参数都是可选的.
:arg string url: fetch 的 URL
:arg string method: HTTP 方法, e.g. "GET" or "POST"
:arg headers: 额外的 HTTP 请求头
:type headers: `~tornado.httputil.HTTPHeaders` 或 `dict`
:arg body: HTTP 请求体字符串 (byte 或 unicode; 如果是 unicode
则使用 utf-8 编码)
:arg body_producer: 可以被用于延迟/异步请求体调用.
它可以被调用, 带有一个参数, 一个 ``write`` 函数, 并应该
返回一个 `.Future` 对象. 它应该在新的数据可用时调用 write 函数.
write 函数返回一个可用于流程控制的 `.Future` 对象.
只能指定 ``body`` 和 ``body_producer`` 其中之一.
``body_producer`` 不被 ``curl_httpclient`` 支持.
当使用 ``body_producer`` 时, 建议传递一个
``Content-Length`` 头, 否则将使用其他的分块编码,
并且很多服务断不支持请求的分块编码. Tornado 4.0 新增
:arg string auth_username: HTTP 认证的用户名
:arg string auth_password: HTTP 认证的密码
:arg string auth_mode: 认证模式; 默认是 "basic".
所允许的值是根据实现方式定义的; ``curl_httpclient``
支持 "basic" 和 "digest"; ``simple_httpclient`` 只支持 "basic"
:arg float connect_timeout: 初始化连接的超时时间
:arg float request_timeout: 整个请求的超时时间
:arg if_modified_since: ``If-Modified-Since`` 头的时间戳
:type if_modified_since: `datetime` 或 `float`
:arg bool follow_redirects: 是否应该自动跟随重定向还是返回 3xx 响应?
:arg int max_redirects: ``follow_redirects`` 的最大次数限制
:arg string user_agent: ``User-Agent`` 头
:arg bool decompress_response: 从服务器请求一个压缩过的响应, 在下载
后对其解压缩. 默认是 True.
Tornado 4.0 新增.
:arg bool use_gzip: ``decompress_response`` 的别名从 Tornado 4.0 已弃用.
:arg string network_interface: 请求所使用的网络接口.
只有 ``curl_httpclient`` ; 请看下面的备注.
:arg callable streaming_callback: 如果设置了, ``streaming_callback`` 将
用它接收到的数据块执行, 并且
``HTTPResponse.body`` 和 ``HTTPResponse.buffer`` 在最后的响应中将为空.
:arg callable header_callback: 如果设置了, ``header_callback`` 将
在接收到每行头信息时运行(包括第一行, e.g. ``HTTP/1.0 200 OK\r\n``,
最后一行只包含 ``\r\n``. 所有行都包含结尾的换行符).
``HTTPResponse.headers`` 在最终响应中将为空. 这与
``streaming_callback`` 结合是最有用的, 因为它是在请求正在进行时
访问头信息唯一的方法.
:arg callable prepare_curl_callback: 如果设置, 将使用
``pycurl.Curl`` 对象调用, 以允许应用程序进行额外的
``setopt`` 调用.
:arg string proxy_host: HTTP 代理主机名. 如果想要使用代理,
``proxy_host`` 和 ``proxy_port`` 必须设置; ``proxy_username`` 和
``proxy_pass`` 是可选项. 目前只有 ``curl_httpclient`` 支持代理.
:arg int proxy_port: HTTP 代理端口
:arg string proxy_username: HTTP 代理用户名
:arg string proxy_password: HTTP 代理密码
:arg bool allow_nonstandard_methods: 允许 ``method`` 参数使用未知值?
:arg bool validate_cert: 对于 HTTPS 请求, 是否验证服务器的证书?
:arg string ca_certs: PEM 格式的 CA 证书的文件名, 或者默认为 None.
当与 ``curl_httpclient`` 一起使用时参阅下面的注释.
:arg string client_key: 客户端 SSL key 文件名(如果有).
当与 ``curl_httpclient`` 一起使用时参阅下面的注释.
:arg string client_cert: 客户端 SSL 证书的文件名(如果有).
当与 ``curl_httpclient`` 一起使用时参阅下面的注释.
:arg ssl.SSLContext ssl_options: 用在
``simple_httpclient`` (``curl_httpclient`` 不支持) 的
`ssl.SSLContext` 对象.
覆写 ``validate_cert``, ``ca_certs``, ``client_key``,
和 ``client_cert``.
:arg bool allow_ipv6: 当 IPv6 可用时是否使用? 默认是 true.
:arg bool expect_100_continue: 如果为 true, 发送
``Expect: 100-continue`` 头并在发送请求体前等待继续响应.
只被 simple_httpclient 支持.
.. 注意::
当使用 ``curl_httpclient`` 时, 某些选项可能会被后续获取
的继承, 因为 ``pycurl`` 不允许它们被彻底重置. 这适用于
``ca_certs``, ``client_key``, ``client_cert``, 和
``network_interface`` 参数. 如果你使用这些参数, 你应该在
每次请求中都传递它们(你不必总使用相同的值, 但不能混合
指定了这些参数和使用默认参数的请求).
.. versionadded:: 3.1
``auth_mode`` 参数.
.. versionadded:: 4.0
``body_producer`` 和 ``expect_100_continue`` 参数.
.. versionadded:: 4.2
``ssl_options`` 参数.
"""
# Note that some of these attributes go through property setters
# defined below.
self.headers = headers
if if_modified_since:
self.headers["If-Modified-Since"] = httputil.format_timestamp(
if_modified_since)
self.proxy_host = proxy_host
self.proxy_port = proxy_port
self.proxy_username = proxy_username
self.proxy_password = proxy_password
self.url = url
self.method = method
self.body = body
self.body_producer = body_producer
self.auth_username = auth_username
self.auth_password = auth_password
self.auth_mode = auth_mode
self.connect_timeout = connect_timeout
self.request_timeout = request_timeout
self.follow_redirects = follow_redirects
self.max_redirects = max_redirects
self.user_agent = user_agent
if decompress_response is not None:
self.decompress_response = decompress_response
else:
self.decompress_response = use_gzip
self.network_interface = network_interface
self.streaming_callback = streaming_callback
self.header_callback = header_callback
self.prepare_curl_callback = prepare_curl_callback
self.allow_nonstandard_methods = allow_nonstandard_methods
self.validate_cert = validate_cert
self.ca_certs = ca_certs
self.allow_ipv6 = allow_ipv6
self.client_key = client_key
self.client_cert = client_cert
self.ssl_options = ssl_options
self.expect_100_continue = expect_100_continue
self.start_time = time.time()
