class WsgiResponse: """A WSGI response. Instances are callable using the standard WSGI call and, importantly, iterable:: response = WsgiResponse(200) A :class:`WsgiResponse` is an iterable over bytes to send back to the requesting client. .. attribute:: status_code Integer indicating the HTTP status, (i.e. 200) .. attribute:: response String indicating the HTTP status (i.e. 'OK') .. attribute:: status String indicating the HTTP status code and response (i.e. '200 OK') .. attribute:: content_type The content type of this response. Can be ``None``. .. attribute:: headers The :class:`.Headers` container for this response. .. attribute:: environ The dictionary of WSGI environment if passed to the constructor. .. attribute:: cookies A python :class:`SimpleCookie` container of cookies included in the request as well as cookies set during the response. """ _iterated = False _started = False DEFAULT_STATUS_CODE = 200 def __init__(self, status=None, content=None, response_headers=None, content_type=None, encoding=None, environ=None, can_store_cookies=True): self.environ = environ self.status_code = status or self.DEFAULT_STATUS_CODE self.encoding = encoding self.cookies = SimpleCookie() self.headers = Headers(response_headers, kind='server') self.content = content self._can_store_cookies = can_store_cookies if content_type is not None: self.content_type = content_type @property def started(self): return self._started @property def iterated(self): return self._iterated @property def path(self): if self.environ: return self.environ.get('PATH_INFO', '') @property def method(self): if self.environ: return self.environ.get('REQUEST_METHOD') @property def connection(self): if self.environ: return self.environ.get('pulsar.connection') @property def content(self): return self._content @content.setter def content(self, content): if not self._iterated: if content is None: content = () else: if isinstance(content, str): if not self.encoding: # use utf-8 if not set self.encoding = 'utf-8' content = content.encode(self.encoding) if isinstance(content, bytes): content = (content, ) self._content = content else: raise RuntimeError('Cannot set content. Already iterated') def _get_content_type(self): return self.headers.get('content-type') def _set_content_type(self, typ): if typ: self.headers['content-type'] = typ else: self.headers.pop('content-type', None) content_type = property(_get_content_type, _set_content_type) @property def response(self): return responses.get(self.status_code) @property def status(self): return '%s %s' % (self.status_code, self.response) def __str__(self): return self.status def __repr__(self): return '%s(%s)' % (self.__class__.__name__, self) @property def is_streamed(self): """Check if the response is streamed. A streamed response is an iterable with no length information. In this case streamed means that there is no information about the number of iterations. This is usually `True` if a generator is passed to the response object. """ try: len(self.content) except TypeError: return True return False def can_set_cookies(self): if self.status_code < 400: return self._can_store_cookies def length(self): if not self.is_streamed: return reduce(lambda x, y: x + len(y), self.content, 0) def start(self, start_response): assert not self._started self._started = True return start_response(self.status, self.get_headers()) def __iter__(self): if self._iterated: raise RuntimeError('WsgiResponse can be iterated once only') self._started = True self._iterated = True if self.is_streamed: return wsgi_encoder(self.content, self.encoding or 'utf-8') else: return iter(self.content) def close(self): """Close this response, required by WSGI """ if self.is_streamed: if hasattr(self.content, 'close'): self.content.close() def set_cookie(self, key, **kwargs): """ Sets a cookie. ``expires`` can be a string in the correct format or a ``datetime.datetime`` object in UTC. If ``expires`` is a datetime object then ``max_age`` will be calculated. """ set_cookie(self.cookies, key, **kwargs) def delete_cookie(self, key, path='/', domain=None): set_cookie(self.cookies, key, max_age=0, path=path, domain=domain, expires='Thu, 01-Jan-1970 00:00:00 GMT') def get_headers(self): """The list of headers for this response """ headers = self.headers if has_empty_content(self.status_code): headers.pop('content-type', None) headers.pop('content-length', None) self._content = () else: if not self.is_streamed: cl = 0 for c in self.content: cl += len(c) if cl == 0 and self.content_type in JSON_CONTENT_TYPES: self._content = (b'{}', ) cl = len(self._content[0]) headers['Content-Length'] = str(cl) ct = self.content_type # content type encoding available if self.encoding: ct = ct or 'text/plain' if 'charset=' not in ct: ct = '%s; charset=%s' % (ct, self.encoding) if ct: headers['Content-Type'] = ct if self.method == HEAD: self._content = () if self.can_set_cookies(): for c in self.cookies.values(): headers.add_header('Set-Cookie', c.OutputString()) return list(headers) def has_header(self, header): return header in self.headers __contains__ = has_header def __setitem__(self, header, value): self.headers[header] = value def __getitem__(self, header): return self.headers[header]
class HttpRequest(RequestBase): '''An :class:`HttpClient` request for an HTTP resource. This class has a similar interface to :class:`urllib.request.Request`. :param files: optional dictionary of name, file-like-objects. :param allow_redirects: allow the response to follow redirects. .. attribute:: method The request method .. attribute:: version HTTP version for this request, usually ``HTTP/1.1`` .. attribute:: encode_multipart If ``True`` (default), defaults POST data as ``multipart/form-data``. Pass ``encode_multipart=False`` to default to ``application/x-www-form-urlencoded``. In any case, this parameter is overwritten by passing the correct content-type header. .. attribute:: history List of past :class:`.HttpResponse` (collected during redirects). .. attribute:: wait_continue if ``True``, the :class:`HttpRequest` includes the ``Expect: 100-Continue`` header. ''' CONNECT = 'CONNECT' _proxy = None _ssl = None _tunnel = None def __init__(self, client, url, method, inp_params=None, headers=None, data=None, files=None, history=None, charset=None, encode_multipart=True, multipart_boundary=None, source_address=None, allow_redirects=False, max_redirects=10, decompress=True, version=None, wait_continue=False, websocket_handler=None, cookies=None, urlparams=None, **ignored): self.client = client self._data = None self.files = files self.urlparams = urlparams self.inp_params = inp_params or {} self.unredirected_headers = Headers(kind='client') self.method = method.upper() self.full_url = url if urlparams: self._encode_url(urlparams) self.set_proxy(None) self.history = history self.wait_continue = wait_continue self.max_redirects = max_redirects self.allow_redirects = allow_redirects self.charset = charset or 'utf-8' self.version = version self.decompress = decompress self.encode_multipart = encode_multipart self.multipart_boundary = multipart_boundary self.websocket_handler = websocket_handler self.source_address = source_address self.new_parser() if self._scheme in tls_schemes: self._ssl = client.ssl_context(**ignored) self.headers = client.get_headers(self, headers) cookies = cookiejar_from_dict(client.cookies, cookies) if cookies: cookies.add_cookie_header(self) self.unredirected_headers['host'] = host_no_default_port(self._scheme, self._netloc) client.set_proxy(self) self.data = data @property def address(self): '''``(host, port)`` tuple of the HTTP resource''' return self._tunnel.address if self._tunnel else (self.host, self.port) @property def target_address(self): return (self.host, int(self.port)) @property def ssl(self): '''Context for TLS connections. If this is a tunneled request and the tunnel connection is not yet established, it returns ``None``. ''' if not self._tunnel: return self._ssl @property def key(self): return (self.scheme, self.host, self.port) @property def proxy(self): '''Proxy server for this request.''' return self._proxy @property def netloc(self): if self._proxy: return self._proxy.netloc else: return self._netloc def __repr__(self): return self.first_line() __str__ = __repr__ @property def full_url(self): '''Full url of endpoint''' return urlunparse((self._scheme, self._netloc, self.path, self.params, self.query, self.fragment)) @full_url.setter def full_url(self, url): self._scheme, self._netloc, self.path, self.params,\ self.query, self.fragment = urlparse(url) if not self._netloc and self.method == 'CONNECT': self._scheme, self._netloc, self.path, self.params,\ self.query, self.fragment = urlparse('http://%s' % url) @property def data(self): '''Body of request''' return self._data @data.setter def data(self, data): self._data = self._encode_data(data) def first_line(self): if not self._proxy and self.method != self.CONNECT: url = urlunparse(('', '', self.path or '/', self.params, self.query, self.fragment)) else: url = self.full_url return '%s %s %s' % (self.method, url, self.version) def new_parser(self): self.parser = self.client.http_parser(kind=1, decompress=self.decompress, method=self.method) def set_proxy(self, scheme, *host): if not host and scheme is None: self.scheme = self._scheme self._set_hostport(self._scheme, self._netloc) else: le = 2 + len(host) if not le == 3: raise TypeError( 'set_proxy() takes exactly three arguments (%s given)' % le) if not self._ssl: self.scheme = scheme self._set_hostport(scheme, host[0]) self._proxy = scheme_host(scheme, host[0]) else: self._tunnel = HttpTunnel(self, scheme, host[0]) def _set_hostport(self, scheme, host): self._tunnel = None self._proxy = None self.host, self.port = get_hostport(scheme, host) def encode(self): '''The bytes representation of this :class:`HttpRequest`. Called by :class:`HttpResponse` when it needs to encode this :class:`HttpRequest` before sending it to the HTTP resource. ''' # Call body before fist_line in case the query is changes. first_line = self.first_line() body = self.data if body and self.wait_continue: self.headers['expect'] = '100-continue' body = None headers = self.headers if self.unredirected_headers: headers = self.unredirected_headers.copy() headers.update(self.headers) buffer = [first_line.encode('ascii'), b'\r\n', bytes(headers)] if body: buffer.append(body) return b''.join(buffer) def add_header(self, key, value): self.headers[key] = value def has_header(self, header_name): '''Check ``header_name`` is in this request headers. ''' return (header_name in self.headers or header_name in self.unredirected_headers) def get_header(self, header_name, default=None): '''Retrieve ``header_name`` from this request headers. ''' return self.headers.get( header_name, self.unredirected_headers.get(header_name, default)) def remove_header(self, header_name): '''Remove ``header_name`` from this request. ''' self.headers.pop(header_name, None) self.unredirected_headers.pop(header_name, None) def add_unredirected_header(self, header_name, header_value): self.unredirected_headers[header_name] = header_value # INTERNAL ENCODING METHODS def _encode_data(self, data): body = None if self.method in ENCODE_URL_METHODS: self.files = None self._encode_url(data) elif isinstance(data, bytes): assert self.files is None, ('data cannot be bytes when files are ' 'present') body = data elif isinstance(data, str): assert self.files is None, ('data cannot be string when files are ' 'present') body = to_bytes(data, self.charset) elif data or self.files: if self.files: body, content_type = self._encode_files(data) else: body, content_type = self._encode_params(data) # set files to None, Important! self.files = None self.headers['Content-Type'] = content_type if body: self.headers['content-length'] = str(len(body)) elif 'expect' not in self.headers: self.headers.pop('content-length', None) self.headers.pop('content-type', None) return body def _encode_url(self, data): query = self.query if data: data = native_str(data) if isinstance(data, str): data = parse_qsl(data) else: data = mapping_iterator(data) query = parse_qsl(query) query.extend(data) query = urlencode(query) self.query = query def _encode_files(self, data): fields = [] for field, val in mapping_iterator(data or ()): if (isinstance(val, str) or isinstance(val, bytes) or not hasattr(val, '__iter__')): val = [val] for v in val: if v is not None: if not isinstance(v, bytes): v = str(v) fields.append((field.decode('utf-8') if isinstance(field, bytes) else field, v.encode('utf-8') if isinstance(v, str) else v)) for (k, v) in mapping_iterator(self.files): # support for explicit filename ft = None if isinstance(v, (tuple, list)): if len(v) == 2: fn, fp = v else: fn, fp, ft = v else: fn = guess_filename(v) or k fp = v if isinstance(fp, bytes): fp = BytesIO(fp) elif isinstance(fp, str): fp = StringIO(fp) if ft: new_v = (fn, fp.read(), ft) else: new_v = (fn, fp.read()) fields.append((k, new_v)) # return encode_multipart_formdata(fields, charset=self.charset) def _encode_params(self, data): content_type = self.headers.get('content-type') # No content type given, chose one if not content_type: if self.encode_multipart: content_type = MULTIPART_FORM_DATA else: content_type = FORM_URL_ENCODED if content_type in JSON_CONTENT_TYPES: body = json.dumps(data).encode(self.charset) elif content_type == FORM_URL_ENCODED: body = urlencode(data).encode(self.charset) elif content_type == MULTIPART_FORM_DATA: body, content_type = encode_multipart_formdata( data, boundary=self.multipart_boundary, charset=self.charset) else: raise ValueError("Don't know how to encode body for %s" % content_type) return body, content_type
class HttpRequest(RequestBase): '''An :class:`HttpClient` request for an HTTP resource. This class has a similar interface to :class:`urllib.request.Request`. :param files: optional dictionary of name, file-like-objects. :param allow_redirects: allow the response to follow redirects. .. attribute:: method The request method .. attribute:: version HTTP version for this request, usually ``HTTP/1.1`` .. attribute:: encode_multipart If ``True`` (default), defaults POST data as ``multipart/form-data``. Pass ``encode_multipart=False`` to default to ``application/x-www-form-urlencoded``. In any case, this parameter is overwritten by passing the correct content-type header. .. attribute:: history List of past :class:`.HttpResponse` (collected during redirects). .. attribute:: wait_continue if ``True``, the :class:`HttpRequest` includes the ``Expect: 100-Continue`` header. ''' CONNECT = 'CONNECT' _proxy = None _ssl = None _tunnel = None def __init__(self, client, url, method, inp_params=None, headers=None, data=None, files=None, history=None, charset=None, encode_multipart=True, multipart_boundary=None, source_address=None, allow_redirects=False, max_redirects=10, decompress=True, version=None, wait_continue=False, websocket_handler=None, cookies=None, urlparams=None, **ignored): self.client = client self._data = None self.files = files self.urlparams = urlparams self.inp_params = inp_params or {} self.unredirected_headers = Headers(kind='client') self.method = method.upper() self.full_url = url if urlparams: self._encode_url(urlparams) self.set_proxy(None) self.history = history self.wait_continue = wait_continue self.max_redirects = max_redirects self.allow_redirects = allow_redirects self.charset = charset or 'utf-8' self.version = version self.decompress = decompress self.encode_multipart = encode_multipart self.multipart_boundary = multipart_boundary self.websocket_handler = websocket_handler self.source_address = source_address self.new_parser() if self._scheme in tls_schemes: self._ssl = client.ssl_context(**ignored) self.headers = client.get_headers(self, headers) cookies = cookiejar_from_dict(client.cookies, cookies) if cookies: cookies.add_cookie_header(self) self.unredirected_headers['host'] = host_no_default_port( self._scheme, self._netloc) client.set_proxy(self) self.data = data @property def address(self): '''``(host, port)`` tuple of the HTTP resource''' return self._tunnel.address if self._tunnel else (self.host, self.port) @property def target_address(self): return (self.host, int(self.port)) @property def ssl(self): '''Context for TLS connections. If this is a tunneled request and the tunnel connection is not yet established, it returns ``None``. ''' if not self._tunnel: return self._ssl @property def key(self): return (self.scheme, self.host, self.port) @property def proxy(self): '''Proxy server for this request.''' return self._proxy @property def netloc(self): if self._proxy: return self._proxy.netloc else: return self._netloc def __repr__(self): return self.first_line() __str__ = __repr__ @property def full_url(self): '''Full url of endpoint''' return urlunparse((self._scheme, self._netloc, self.path, self.params, self.query, self.fragment)) @full_url.setter def full_url(self, url): self._scheme, self._netloc, self.path, self.params,\ self.query, self.fragment = urlparse(url) if not self._netloc and self.method == 'CONNECT': self._scheme, self._netloc, self.path, self.params,\ self.query, self.fragment = urlparse('http://%s' % url) @property def data(self): '''Body of request''' return self._data @data.setter def data(self, data): self._data = self._encode_data(data) def first_line(self): if not self._proxy and self.method != self.CONNECT: url = urlunparse(('', '', self.path or '/', self.params, self.query, self.fragment)) else: url = self.full_url return '%s %s %s' % (self.method, url, self.version) def new_parser(self): self.parser = self.client.http_parser(kind=1, decompress=self.decompress, method=self.method) def set_proxy(self, scheme, *host): if not host and scheme is None: self.scheme = self._scheme self._set_hostport(self._scheme, self._netloc) else: le = 2 + len(host) if not le == 3: raise TypeError( 'set_proxy() takes exactly three arguments (%s given)' % le) if not self._ssl: self.scheme = scheme self._set_hostport(scheme, host[0]) self._proxy = scheme_host(scheme, host[0]) else: self._tunnel = HttpTunnel(self, scheme, host[0]) def _set_hostport(self, scheme, host): self._tunnel = None self._proxy = None self.host, self.port = get_hostport(scheme, host) def encode(self): '''The bytes representation of this :class:`HttpRequest`. Called by :class:`HttpResponse` when it needs to encode this :class:`HttpRequest` before sending it to the HTTP resource. ''' # Call body before fist_line in case the query is changes. first_line = self.first_line() body = self.data if body and self.wait_continue: self.headers['expect'] = '100-continue' body = None headers = self.headers if self.unredirected_headers: headers = self.unredirected_headers.copy() headers.update(self.headers) buffer = [first_line.encode('ascii'), b'\r\n', bytes(headers)] if body: buffer.append(body) return b''.join(buffer) def add_header(self, key, value): self.headers[key] = value def has_header(self, header_name): '''Check ``header_name`` is in this request headers. ''' return (header_name in self.headers or header_name in self.unredirected_headers) def get_header(self, header_name, default=None): '''Retrieve ``header_name`` from this request headers. ''' return self.headers.get( header_name, self.unredirected_headers.get(header_name, default)) def remove_header(self, header_name): '''Remove ``header_name`` from this request. ''' self.headers.pop(header_name, None) self.unredirected_headers.pop(header_name, None) def add_unredirected_header(self, header_name, header_value): self.unredirected_headers[header_name] = header_value # INTERNAL ENCODING METHODS def _encode_data(self, data): body = None if self.method in ENCODE_URL_METHODS: self.files = None self._encode_url(data) elif isinstance(data, bytes): assert self.files is None, ('data cannot be bytes when files are ' 'present') body = data elif isinstance(data, str): assert self.files is None, ('data cannot be string when files are ' 'present') body = to_bytes(data, self.charset) elif data or self.files: if self.files: body, content_type = self._encode_files(data) else: body, content_type = self._encode_params(data) # set files to None, Important! self.files = None self.headers['Content-Type'] = content_type if body: self.headers['content-length'] = str(len(body)) elif 'expect' not in self.headers: self.headers.pop('content-length', None) self.headers.pop('content-type', None) return body def _encode_url(self, data): query = self.query if data: data = native_str(data) if isinstance(data, str): data = parse_qsl(data) else: data = mapping_iterator(data) query = parse_qsl(query) query.extend(data) query = urlencode(query) self.query = query def _encode_files(self, data): fields = [] for field, val in mapping_iterator(data or ()): if (isinstance(val, str) or isinstance(val, bytes) or not hasattr(val, '__iter__')): val = [val] for v in val: if v is not None: if not isinstance(v, bytes): v = str(v) fields.append( (field.decode('utf-8') if isinstance(field, bytes) else field, v.encode('utf-8') if isinstance(v, str) else v)) for (k, v) in mapping_iterator(self.files): # support for explicit filename ft = None if isinstance(v, (tuple, list)): if len(v) == 2: fn, fp = v else: fn, fp, ft = v else: fn = guess_filename(v) or k fp = v if isinstance(fp, bytes): fp = BytesIO(fp) elif isinstance(fp, str): fp = StringIO(fp) if ft: new_v = (fn, fp.read(), ft) else: new_v = (fn, fp.read()) fields.append((k, new_v)) # return encode_multipart_formdata(fields, charset=self.charset) def _encode_params(self, data): content_type = self.headers.get('content-type') # No content type given, chose one if not content_type: if self.encode_multipart: content_type = MULTIPART_FORM_DATA else: content_type = FORM_URL_ENCODED if content_type in JSON_CONTENT_TYPES: body = json.dumps(data).encode(self.charset) elif content_type == FORM_URL_ENCODED: body = urlencode(data).encode(self.charset) elif content_type == MULTIPART_FORM_DATA: body, content_type = encode_multipart_formdata( data, boundary=self.multipart_boundary, charset=self.charset) else: raise ValueError("Don't know how to encode body for %s" % content_type) return body, content_type
class HttpRequest(RequestBase): """An :class:`HttpClient` request for an HTTP resource. This class has a similar interface to :class:`urllib.request.Request`. :param files: optional dictionary of name, file-like-objects. :param allow_redirects: allow the response to follow redirects. .. attribute:: method The request method .. attribute:: version HTTP version for this request, usually ``HTTP/1.1`` .. attribute:: history List of past :class:`.HttpResponse` (collected during redirects). .. attribute:: wait_continue if ``True``, the :class:`HttpRequest` includes the ``Expect: 100-Continue`` header. .. attribute:: stream Allow for streaming body """ _proxy = None _ssl = None _tunnel = None _write_done = False def __init__(self, client, url, method, inp_params=None, headers=None, data=None, files=None, json=None, history=None, auth=None, charset=None, max_redirects=10, source_address=None, allow_redirects=False, decompress=True, version=None, wait_continue=False, websocket_handler=None, cookies=None, params=None, stream=False, proxies=None, verify=True, **ignored): self.client = client self.method = method.upper() self.inp_params = inp_params or {} self.unredirected_headers = Headers() self.history = history self.wait_continue = wait_continue self.max_redirects = max_redirects self.allow_redirects = allow_redirects self.charset = charset or 'utf-8' self.version = version self.decompress = decompress self.websocket_handler = websocket_handler self.source_address = source_address self.stream = stream self.verify = verify self.new_parser() if auth and not isinstance(auth, Auth): auth = HTTPBasicAuth(*auth) self.auth = auth self.headers = client._get_headers(headers) self.url = full_url(url, params, method=self.method) self.body = self._encode_body(data, files, json) self._set_proxy(proxies, ignored) cookies = cookiejar_from_dict(client.cookies, cookies) if cookies: cookies.add_cookie_header(self) @property def _loop(self): return self.client._loop @property def address(self): """``(host, port)`` tuple of the HTTP resource """ return self._tunnel.address if self._tunnel else super().address @property def ssl(self): """Context for TLS connections. If this is a tunneled request and the tunnel connection is not yet established, it returns ``None``. """ if not self._tunnel: return self._ssl @property def key(self): tunnel = self._tunnel.url if self._tunnel else None scheme, host, port = scheme_host_port(self._proxy or self.url) return scheme, host, port, tunnel, self.verify @property def proxy(self): """Proxy server for this request. """ return self._proxy @property def tunnel(self): """Tunnel for this request. """ return self._tunnel def __repr__(self): return self.first_line() __str__ = __repr__ def first_line(self): p = urlparse(self.url) if self._proxy: if self.method == 'CONNECT': url = p.netloc else: url = self.url else: url = urlunparse(('', '', p.path or '/', p.params, p.query, p.fragment)) return '%s %s %s' % (self.method, url, self.version) def new_parser(self): self.parser = self.client.http_parser( kind=1, decompress=self.decompress ) return self.parser def is_chunked(self): return self.body and 'content-length' not in self.headers def encode(self): """The bytes representation of this :class:`HttpRequest`. Called by :class:`HttpResponse` when it needs to encode this :class:`HttpRequest` before sending it to the HTTP resource. """ # Call body before fist_line in case the query is changes. first_line = self.first_line() if self.body and self.wait_continue: self.headers['expect'] = '100-continue' headers = self.headers if self.unredirected_headers: headers = self.unredirected_headers.copy() headers.update(self.headers) buffer = [first_line.encode('ascii'), b'\r\n', bytes(headers)] return b''.join(buffer) def add_header(self, key, value): self.headers[key] = value def has_header(self, header_name): """Check ``header_name`` is in this request headers. """ return (header_name in self.headers or header_name in self.unredirected_headers) def get_header(self, header_name, default=None): """Retrieve ``header_name`` from this request headers. """ return self.headers.get( header_name, self.unredirected_headers.get(header_name, default)) def remove_header(self, header_name): """Remove ``header_name`` from this request. """ val1 = self.headers.pop(header_name, None) val2 = self.unredirected_headers.pop(header_name, None) return val1 or val2 def add_unredirected_header(self, header_name, header_value): self.unredirected_headers[header_name] = header_value def write_body(self, transport): assert not self._write_done, 'Body already sent' self._write_done = True if not self.body: return if is_streamed(self.body): ensure_future(self._write_streamed_data(transport), loop=self._loop) else: self._write_body_data(transport, self.body, True) # INTERNAL ENCODING METHODS def _encode_body(self, data, files, json): body = None if isinstance(data, (str, bytes)): if files: raise ValueError('data cannot be a string or bytes when ' 'files are present') body = to_bytes(data, self.charset) elif data and is_streamed(data): if files: raise ValueError('data cannot be an iterator when ' 'files are present') if 'content-length' not in self.headers: self.headers['transfer-encoding'] = 'chunked' return data elif data or files: if files: body, content_type = self._encode_files(data, files) else: body, content_type = self._encode_params(data) self.headers['Content-Type'] = content_type elif json: body = _json.dumps(json).encode(self.charset) self.headers['Content-Type'] = 'application/json' if body: self.headers['content-length'] = str(len(body)) return body def _encode_files(self, data, files): fields = [] for field, val in mapping_iterator(data or ()): if (isinstance(val, str) or isinstance(val, bytes) or not hasattr(val, '__iter__')): val = [val] for v in val: if v is not None: if not isinstance(v, bytes): v = str(v) fields.append((field.decode('utf-8') if isinstance(field, bytes) else field, v.encode('utf-8') if isinstance(v, str) else v)) for (k, v) in mapping_iterator(files): # support for explicit filename ft = None if isinstance(v, (tuple, list)): if len(v) == 2: fn, fp = v else: fn, fp, ft = v else: fn = guess_filename(v) or k fp = v if isinstance(fp, bytes): fp = BytesIO(fp) elif isinstance(fp, str): fp = StringIO(fp) if ft: new_v = (fn, fp.read(), ft) else: new_v = (fn, fp.read()) fields.append((k, new_v)) # return encode_multipart_formdata(fields, charset=self.charset) def _encode_params(self, params): content_type = self.headers.get('content-type') # No content type given, chose one if not content_type: content_type = FORM_URL_ENCODED if hasattr(params, 'read'): params = params.read() if content_type in JSON_CONTENT_TYPES: body = _json.dumps(params) elif content_type == FORM_URL_ENCODED: body = urlencode(tuple(split_url_params(params))) elif content_type == MULTIPART_FORM_DATA: body, content_type = encode_multipart_formdata( params, charset=self.charset) else: body = params return to_bytes(body, self.charset), content_type def _write_body_data(self, transport, data, finish=False): if self.is_chunked(): data = http_chunks(data, finish) elif data: data = (data,) else: return for chunk in data: transport.write(chunk) async def _write_streamed_data(self, transport): for data in self.body: if isawaitable(data): data = await data self._write_body_data(transport, data) self._write_body_data(transport, b'', True) # PROXY INTERNALS def _set_proxy(self, proxies, ignored): url = urlparse(self.url) self.unredirected_headers['host'] = host_no_default_port(url.scheme, url.netloc) if url.scheme in tls_schemes: self._ssl = self.client._ssl_context(verify=self.verify, **ignored) request_proxies = self.client.proxies.copy() if proxies: request_proxies.update(proxies) self.proxies = request_proxies # if url.scheme in request_proxies: host, port = get_hostport(url.scheme, url.netloc) no_proxy = [n for n in request_proxies.get('no', '').split(',') if n] if not any(map(host.endswith, no_proxy)): url = request_proxies[url.scheme] if not self._ssl: self._proxy = url else: self._tunnel = HttpTunnel(self, url)
class HttpRequest(RequestBase): """An :class:`HttpClient` request for an HTTP resource. This class has a similar interface to :class:`urllib.request.Request`. :param files: optional dictionary of name, file-like-objects. :param allow_redirects: allow the response to follow redirects. .. attribute:: method The request method .. attribute:: version HTTP version for this request, usually ``HTTP/1.1`` .. attribute:: encode_multipart If ``True`` (default), defaults POST data as ``multipart/form-data``. Pass ``encode_multipart=False`` to default to ``application/x-www-form-urlencoded``. In any case, this parameter is overwritten by passing the correct content-type header. .. attribute:: history List of past :class:`.HttpResponse` (collected during redirects). .. attribute:: wait_continue if ``True``, the :class:`HttpRequest` includes the ``Expect: 100-Continue`` header. .. attribute:: stream Allow for streaming body """ _proxy = None _ssl = None _tunnel = None _write_done = False def __init__(self, client, url, method, inp_params=None, headers=None, data=None, files=None, history=None, auth=None, charset=None, encode_multipart=True, multipart_boundary=None, source_address=None, allow_redirects=False, max_redirects=10, decompress=True, version=None, wait_continue=False, websocket_handler=None, cookies=None, urlparams=None, stream=False, proxies=None, verify=True, **ignored): self.client = client self._data = None self.files = files self.urlparams = urlparams self.inp_params = inp_params or {} self.unredirected_headers = Headers(kind='client') self.method = method.upper() self.full_url = url if urlparams: self._encode_url(urlparams) self.history = history self.wait_continue = wait_continue self.max_redirects = max_redirects self.allow_redirects = allow_redirects self.charset = charset or 'utf-8' self.version = version self.decompress = decompress self.encode_multipart = encode_multipart self.multipart_boundary = multipart_boundary self.websocket_handler = websocket_handler self.source_address = source_address self.stream = stream self.verify = verify self.new_parser() if self._scheme in tls_schemes: self._ssl = client.ssl_context(verify=self.verify, **ignored) if auth and not isinstance(auth, Auth): auth = HTTPBasicAuth(*auth) self.auth = auth self.headers = client.get_headers(self, headers) cookies = cookiejar_from_dict(client.cookies, cookies) if cookies: cookies.add_cookie_header(self) self.unredirected_headers['host'] = host_no_default_port(self._scheme, self._netloc) self.data = data self._set_proxy(proxies) @property def address(self): """``(host, port)`` tuple of the HTTP resource """ return self._tunnel.address if self._tunnel else (self.host, self.port) @property def target_address(self): return (self.host, int(self.port)) @property def ssl(self): """Context for TLS connections. If this is a tunneled request and the tunnel connection is not yet established, it returns ``None``. """ if not self._tunnel: return self._ssl @property def key(self): tunnel = self._tunnel.full_url if self._tunnel else None return (self.scheme, self.host, self.port, tunnel, self.verify) @property def proxy(self): """Proxy server for this request. """ return self._proxy @property def tunnel(self): """Tunnel for this request. """ return self._tunnel @property def netloc(self): if self._proxy: return self._proxy.netloc else: return self._netloc def __repr__(self): return self.first_line() __str__ = __repr__ @property def full_url(self): """Full url of endpoint """ return urlunparse((self._scheme, self._netloc, self.path, self.params, self.query, self.fragment)) @full_url.setter def full_url(self, url): self._scheme, self._netloc, self.path, self.params,\ self.query, self.fragment = urlparse(url) if not self._netloc and self.method == 'CONNECT': self._scheme, self._netloc, self.path, self.params,\ self.query, self.fragment = urlparse('http://%s' % url) @property def data(self): """Body of request """ return self._data @data.setter def data(self, data): self._data = self._encode_data(data) def first_line(self): if self._proxy: if self.method == 'CONNECT': url = self._netloc else: url = self.full_url else: url = urlunparse(('', '', self.path or '/', self.params, self.query, self.fragment)) return '%s %s %s' % (self.method, url, self.version) def new_parser(self): self.parser = self.client.http_parser(kind=1, decompress=self.decompress, method=self.method) def is_chunked(self): return self.data and 'content-length' not in self.headers def encode(self): """The bytes representation of this :class:`HttpRequest`. Called by :class:`HttpResponse` when it needs to encode this :class:`HttpRequest` before sending it to the HTTP resource. """ # Call body before fist_line in case the query is changes. first_line = self.first_line() if self.data and self.wait_continue: self.headers['expect'] = '100-continue' headers = self.headers if self.unredirected_headers: headers = self.unredirected_headers.copy() headers.update(self.headers) buffer = [first_line.encode('ascii'), b'\r\n', bytes(headers)] return b''.join(buffer) def add_header(self, key, value): self.headers[key] = value def has_header(self, header_name): """Check ``header_name`` is in this request headers. """ return (header_name in self.headers or header_name in self.unredirected_headers) def get_header(self, header_name, default=None): """Retrieve ``header_name`` from this request headers. """ return self.headers.get( header_name, self.unredirected_headers.get(header_name, default)) def remove_header(self, header_name): """Remove ``header_name`` from this request. """ val1 = self.headers.pop(header_name, None) val2 = self.unredirected_headers.pop(header_name, None) return val1 or val2 def add_unredirected_header(self, header_name, header_value): self.unredirected_headers[header_name] = header_value def write_body(self, transport): assert not self._write_done, 'Body already sent' self._write_done = True if not self.data: return if is_streamed(self.data): ensure_future(self._write_streamed_data(transport), loop=transport._loop) else: self._write_body_data(transport, self.data, True) # INTERNAL ENCODING METHODS def _encode_data(self, data): body = None if self.method in ENCODE_URL_METHODS: self.files = None self._encode_url(data) elif isinstance(data, bytes): assert self.files is None, ('data cannot be bytes when files are ' 'present') body = data elif isinstance(data, str): assert self.files is None, ('data cannot be string when files are ' 'present') body = to_bytes(data, self.charset) elif data and is_streamed(data): assert self.files is None, ('data cannot be an iterator when ' 'files are present') if 'content-type' not in self.headers: self.headers['content-type'] = 'application/octet-stream' if 'content-length' not in self.headers: self.headers['transfer-encoding'] = 'chunked' return data elif data or self.files: if self.files: body, content_type = self._encode_files(data) else: body, content_type = self._encode_params(data) # set files to None, Important! self.files = None self.headers['Content-Type'] = content_type if body: self.headers['content-length'] = str(len(body)) elif 'expect' not in self.headers: self.headers.pop('content-length', None) self.headers.pop('content-type', None) return body def _encode_url(self, data): query = self.query if data: data = native_str(data) if isinstance(data, str): data = parse_qsl(data) else: data = mapping_iterator(data) query = parse_qsl(query) query.extend(data) query = urlencode(query) self.query = query def _encode_files(self, data): fields = [] for field, val in mapping_iterator(data or ()): if (isinstance(val, str) or isinstance(val, bytes) or not hasattr(val, '__iter__')): val = [val] for v in val: if v is not None: if not isinstance(v, bytes): v = str(v) fields.append((field.decode('utf-8') if isinstance(field, bytes) else field, v.encode('utf-8') if isinstance(v, str) else v)) for (k, v) in mapping_iterator(self.files): # support for explicit filename ft = None if isinstance(v, (tuple, list)): if len(v) == 2: fn, fp = v else: fn, fp, ft = v else: fn = guess_filename(v) or k fp = v if isinstance(fp, bytes): fp = BytesIO(fp) elif isinstance(fp, str): fp = StringIO(fp) if ft: new_v = (fn, fp.read(), ft) else: new_v = (fn, fp.read()) fields.append((k, new_v)) # return encode_multipart_formdata(fields, charset=self.charset) def _encode_params(self, data): content_type = self.headers.get('content-type') # No content type given, chose one if not content_type: if self.encode_multipart: content_type = MULTIPART_FORM_DATA else: content_type = FORM_URL_ENCODED if content_type in JSON_CONTENT_TYPES: body = json.dumps(data).encode(self.charset) elif content_type == FORM_URL_ENCODED: body = urlencode(data).encode(self.charset) elif content_type == MULTIPART_FORM_DATA: body, content_type = encode_multipart_formdata( data, boundary=self.multipart_boundary, charset=self.charset) else: raise ValueError("Don't know how to encode body for %s" % content_type) return body, content_type def _write_body_data(self, transport, data, finish=False): if self.is_chunked(): data = http_chunks(data, finish) elif data: data = (data,) else: return for chunk in data: transport.write(chunk) @asyncio.coroutine def _write_streamed_data(self, transport): for data in self.data: if isawaitable(data): data = yield from data self._write_body_data(transport, data) self._write_body_data(transport, b'', True) # PROXY INTERNALS def _set_proxy(self, proxies): request_proxies = self.client.proxies.copy() if proxies: request_proxies.update(proxies) self.proxies = request_proxies self.scheme = self._scheme self._set_hostport(self._scheme, self._netloc) # if self.scheme in request_proxies: hostonly = self.host no_proxy = [n for n in request_proxies.get('no', '').split(',') if n] if not any(map(hostonly.endswith, no_proxy)): url = request_proxies[self.scheme] p = urlparse(url) if not p.scheme: raise ValueError('Could not understand proxy %s' % url) scheme = p.scheme host = p.netloc if not self._ssl: self.scheme = scheme self._set_hostport(scheme, host) self._proxy = scheme_host(scheme, host) else: self._tunnel = HttpTunnel(self, scheme, host) def _set_hostport(self, scheme, host): self._tunnel = None self._proxy = None self.host, self.port = get_hostport(scheme, host)
class WsgiResponse: """A WSGI response. Instances are callable using the standard WSGI call and, importantly, iterable:: response = WsgiResponse(200) A :class:`WsgiResponse` is an iterable over bytes to send back to the requesting client. .. attribute:: status_code Integer indicating the HTTP status, (i.e. 200) .. attribute:: response String indicating the HTTP status (i.e. 'OK') .. attribute:: status String indicating the HTTP status code and response (i.e. '200 OK') .. attribute:: content_type The content type of this response. Can be ``None``. .. attribute:: headers The :class:`.Headers` container for this response. .. attribute:: environ The dictionary of WSGI environment if passed to the constructor. .. attribute:: cookies A python :class:`SimpleCookie` container of cookies included in the request as well as cookies set during the response. """ _iterated = False _started = False DEFAULT_STATUS_CODE = 200 def __init__(self, status=None, content=None, response_headers=None, content_type=None, encoding=None, environ=None, can_store_cookies=True): self.environ = environ self.status_code = status or self.DEFAULT_STATUS_CODE self.encoding = encoding self.cookies = SimpleCookie() self.headers = Headers(response_headers, kind='server') self.content = content self._can_store_cookies = can_store_cookies if content_type is not None: self.content_type = content_type @property def started(self): return self._started @property def iterated(self): return self._iterated @property def path(self): if self.environ: return self.environ.get('PATH_INFO', '') @property def method(self): if self.environ: return self.environ.get('REQUEST_METHOD') @property def connection(self): if self.environ: return self.environ.get('pulsar.connection') @property def content(self): return self._content @content.setter def content(self, content): if not self._iterated: if content is None: content = () else: if isinstance(content, str): if not self.encoding: # use utf-8 if not set self.encoding = 'utf-8' content = content.encode(self.encoding) if isinstance(content, bytes): content = (content,) self._content = content else: raise RuntimeError('Cannot set content. Already iterated') def _get_content_type(self): return self.headers.get('content-type') def _set_content_type(self, typ): if typ: self.headers['content-type'] = typ else: self.headers.pop('content-type', None) content_type = property(_get_content_type, _set_content_type) @property def response(self): return responses.get(self.status_code) @property def status(self): return '%s %s' % (self.status_code, self.response) def __str__(self): return self.status def __repr__(self): return '%s(%s)' % (self.__class__.__name__, self) @property def is_streamed(self): """Check if the response is streamed. A streamed response is an iterable with no length information. In this case streamed means that there is no information about the number of iterations. This is usually `True` if a generator is passed to the response object. """ try: len(self.content) except TypeError: return True return False def can_set_cookies(self): if self.status_code < 400: return self._can_store_cookies def length(self): if not self.is_streamed: return reduce(lambda x, y: x+len(y), self.content, 0) def start(self, start_response): assert not self._started self._started = True return start_response(self.status, self.get_headers()) def __iter__(self): if self._iterated: raise RuntimeError('WsgiResponse can be iterated once only') self._started = True self._iterated = True if self.is_streamed: return wsgi_encoder(self.content, self.encoding or 'utf-8') else: return iter(self.content) def close(self): """Close this response, required by WSGI """ if self.is_streamed: if hasattr(self.content, 'close'): self.content.close() def set_cookie(self, key, **kwargs): """ Sets a cookie. ``expires`` can be a string in the correct format or a ``datetime.datetime`` object in UTC. If ``expires`` is a datetime object then ``max_age`` will be calculated. """ set_cookie(self.cookies, key, **kwargs) def delete_cookie(self, key, path='/', domain=None): set_cookie(self.cookies, key, max_age=0, path=path, domain=domain, expires='Thu, 01-Jan-1970 00:00:00 GMT') def get_headers(self): """The list of headers for this response """ headers = self.headers if has_empty_content(self.status_code): headers.pop('content-type', None) headers.pop('content-length', None) self._content = () else: if not self.is_streamed: cl = 0 for c in self.content: cl += len(c) if cl == 0 and self.content_type in JSON_CONTENT_TYPES: self._content = (b'{}',) cl = len(self._content[0]) headers['Content-Length'] = str(cl) ct = self.content_type # content type encoding available if self.encoding: ct = ct or 'text/plain' if 'charset=' not in ct: ct = '%s; charset=%s' % (ct, self.encoding) if ct: headers['Content-Type'] = ct if self.method == HEAD: self._content = () if self.can_set_cookies(): for c in self.cookies.values(): headers.add_header('Set-Cookie', c.OutputString()) return list(headers) def has_header(self, header): return header in self.headers __contains__ = has_header def __setitem__(self, header, value): self.headers[header] = value def __getitem__(self, header): return self.headers[header]
class WsgiResponse(WsgiResponseGenerator): """A WSGI response wrapper initialized by a WSGI request middleware. Instances are callable using the standard WSGI call:: response = WsgiResponse(200) response(environ, start_response) A :class:`WsgiResponse` is an iterable over bytes to send back to the requesting client. .. attribute:: status_code Integer indicating the HTTP status, (i.e. 200) .. attribute:: response String indicating the HTTP status (i.e. 'OK') .. attribute:: status String indicating the HTTP status code and response (i.e. '200 OK') .. attribute:: environ The dictionary of WSGI environment if passed to the constructor. """ _started = False DEFAULT_STATUS_CODE = 200 def __init__( self, status=None, content=None, response_headers=None, content_type=None, encoding=None, environ=None, start_response=None, ): super(WsgiResponse, self).__init__(environ, start_response) self.status_code = status or self.DEFAULT_STATUS_CODE self.encoding = encoding self.cookies = SimpleCookie() self.headers = Headers(response_headers, kind="server") self.content = content if content_type is not None: self.content_type = content_type @property def started(self): return self._started @property def path(self): if self.environ: return self.environ.get("PATH_INFO", "") @property def method(self): if self.environ: return self.environ.get("REQUEST_METHOD") @property def connection(self): if self.environ: return self.environ.get("pulsar.connection") def _get_content(self): return self._content def _set_content(self, content): if not self._started: if content is None: content = () elif ispy3k: # what a f*****g pain if isinstance(content, str): content = bytes(content, "latin-1") else: # pragma nocover if isinstance(content, unicode): content = bytes(content, "latin-1") if isinstance(content, bytes): content = (content,) self._content = content else: raise RuntimeError("Cannot set content. Already iterated") content = property(_get_content, _set_content) def _get_content_type(self): return self.headers.get("content-type") def _set_content_type(self, typ): if typ: self.headers["content-type"] = typ else: self.headers.pop("content-type", None) content_type = property(_get_content_type, _set_content_type) def __call__(self, environ, start_response, exc_info=None): """Make sure the headers are set.""" if not exc_info: for rm in self.middleware: try: rm(environ, self) except: LOGGER.error("Exception in response middleware", exc_info=True) start_response(self.status, self.get_headers(), exc_info=exc_info) return self def start(self): self.__call__(self.environ, self.start_response) def length(self): if not self.is_streamed: return reduce(lambda x, y: x + len(y), self.content, 0) @property def response(self): return responses.get(self.status_code) @property def status(self): return "%s %s" % (self.status_code, self.response) def __str__(self): return self.status def __repr__(self): return "%s(%s)" % (self.__class__.__name__, self) @property def is_streamed(self): """If the response is streamed (the response is not an iterable with length information) this property is `True`. In this case streamed means that there is no information about the number of iterations. This is usually `True` if a generator is passed to the response object.""" return is_streamed(self.content) def __iter__(self): if self._started: raise RuntimeError("WsgiResponse can be iterated only once") self._started = True if self.is_streamed: return wsgi_iterator(self.content, self.encoding) else: return iter(self.content) def __len__(self): return len(self.content) def set_cookie(self, key, **kwargs): """ Sets a cookie. ``expires`` can be a string in the correct format or a ``datetime.datetime`` object in UTC. If ``expires`` is a datetime object then ``max_age`` will be calculated. """ set_cookie(self.cookies, key, **kwargs) def delete_cookie(self, key, path="/", domain=None): set_cookie(self.cookies, key, max_age=0, path=path, domain=domain, expires="Thu, 01-Jan-1970 00:00:00 GMT") def get_headers(self): headers = self.headers if has_empty_content(self.status_code, self.method): headers.pop("content-type", None) headers.pop("content-length", None) elif not self.is_streamed: cl = 0 for c in self.content: cl += len(c) headers["Content-Length"] = str(cl) for c in self.cookies.values(): headers["Set-Cookie"] = c.OutputString() return list(headers)
class WsgiResponse(object): '''A WSGI response. Instances are callable using the standard WSGI call and, importantly, iterable:: response = WsgiResponse(200) A :class:`WsgiResponse` is an iterable over bytes to send back to the requesting client. .. attribute:: status_code Integer indicating the HTTP status, (i.e. 200) .. attribute:: response String indicating the HTTP status (i.e. 'OK') .. attribute:: status String indicating the HTTP status code and response (i.e. '200 OK') .. attribute:: content_type The content type of this response. Can be ``None``. .. attribute:: headers The :class:`pulsar.utils.httpurl.Headers` container for this response. .. attribute:: environ The dictionary of WSGI environment if passed to the constructor. ''' _started = False DEFAULT_STATUS_CODE = 200 def __init__(self, status=None, content=None, response_headers=None, content_type=None, encoding=None, environ=None): self.environ = environ self.status_code = status or self.DEFAULT_STATUS_CODE self.encoding = encoding self.cookies = SimpleCookie() self.headers = Headers(response_headers, kind='server') self.content = content if content_type is not None: self.content_type = content_type @property def started(self): return self._started @property def path(self): if self.environ: return self.environ.get('PATH_INFO', '') @property def method(self): if self.environ: return self.environ.get('REQUEST_METHOD') @property def connection(self): if self.environ: return self.environ.get('pulsar.connection') def _get_content(self): return self._content def _set_content(self, content): if not self._started: if content is None: content = () elif ispy3k: if isinstance(content, str): content = content.encode(self.encoding or 'utf-8') else: # pragma nocover if isinstance(content, unicode): content = content.encode(self.encoding or 'utf-8') if isinstance(content, bytes): content = (content,) self._content = content else: raise RuntimeError('Cannot set content. Already iterated') content = property(_get_content, _set_content) def _get_content_type(self): return self.headers.get('content-type') def _set_content_type(self, typ): if typ: self.headers['content-type'] = typ else: self.headers.pop('content-type', None) content_type = property(_get_content_type, _set_content_type) def length(self): if not self.is_streamed: return reduce(lambda x, y: x+len(y), self.content, 0) @property def response(self): return responses.get(self.status_code) @property def status(self): return '%s %s' % (self.status_code, self.response) def __str__(self): return self.status def __repr__(self): return '%s(%s)' % (self.__class__.__name__, self) @property def is_streamed(self): """If the response is streamed (the response is not an iterable with length information) this property is `True`. In this case streamed means that there is no information about the number of iterations. This is usually `True` if a generator is passed to the response object.""" return is_streamed(self.content) def __iter__(self): if self._started: raise RuntimeError('WsgiResponse can be iterated once only') self._started = True if is_streamed(self.content): return wsgi_encoder(self.content, self.encoding or 'utf-8') else: return iter(self.content) def __len__(self): return len(self.content) def set_cookie(self, key, **kwargs): """ Sets a cookie. ``expires`` can be a string in the correct format or a ``datetime.datetime`` object in UTC. If ``expires`` is a datetime object then ``max_age`` will be calculated. """ set_cookie(self.cookies, key, **kwargs) def delete_cookie(self, key, path='/', domain=None): set_cookie(self.cookies, key, max_age=0, path=path, domain=domain, expires='Thu, 01-Jan-1970 00:00:00 GMT') def get_headers(self): headers = self.headers if has_empty_content(self.status_code, self.method): headers.pop('content-type', None) headers.pop('content-length', None) self._content = () else: if not self.is_streamed: cl = 0 for c in self.content: cl += len(c) if cl == 0 and self.content_type in JSON_CONTENT_TYPES: self._content = (b'{}',) cl = len(self._content[0]) headers['Content-Length'] = str(cl) if not self.content_type: headers['Content-Type'] = 'text/plain' for c in self.cookies.values(): headers['Set-Cookie'] = c.OutputString() return list(headers) def has_header(self, header): return header in self.headers __contains__ = has_header def __setitem__(self, header, value): self.headers[header] = value def __getitem__(self, header): return self.headers[header]