class HttpTunnel(RequestBase): first_line = None data = None decompress = False method = 'CONNECT' def __init__(self, client, req): self.client = client self.key = req self.headers = CIMultiDict(client.DEFAULT_TUNNEL_HEADERS) def __repr__(self): return 'Tunnel %s' % self.url __str__ = __repr__ def encode(self): self.headers['host'] = self.key.netloc self.first_line = 'CONNECT http://%s:%s HTTP/1.1' % self.key.address buffer = [self.first_line.encode('ascii'), b'\r\n'] buffer.extend((('%s: %s\r\n' % (name, value)).encode(CHARSET) for name, value in self.headers.items())) buffer.append(b'\r\n') return b''.join(buffer) def has_header(self, header_name): return header_name in self.headers def get_header(self, header_name, default=None): return self.headers.get(header_name, default) def remove_header(self, header_name): self.headers.pop(header_name, None)
class HttpTunnel(RequestBase): first_line = None data = None decompress = False method = 'CONNECT' def __init__(self, client, req): self.client = client self.key = req self.headers = CIMultiDict(client.DEFAULT_TUNNEL_HEADERS) def __repr__(self): return 'Tunnel %s' % self.url __str__ = __repr__ def encode(self): self.headers['host'] = self.key.netloc self.first_line = 'CONNECT http://%s:%s HTTP/1.1' % self.key.address buffer = [self.first_line.encode('ascii'), b'\r\n'] buffer.extend((('%s: %s\r\n' % (name, value)).encode(CHARSET) for name, value in self.headers.items())) buffer.append(b'\r\n') return b''.join(buffer) def has_header(self, header_name): return header_name in self.headers def get_header(self, header_name, default=None): return self.headers.get(header_name, default) def remove_header(self, header_name): self.headers.pop(header_name, None)
def headers_preserve_casing(): try: from multidict import CIMultiDict except: return True d = CIMultiDict() d.update({"X-NewRelic-ID": "value"}) return "X-NewRelic-ID" in dict(d.items())
def headers_preserve_casing(): try: from multidict import CIMultiDict except: return True d = CIMultiDict() d.update({'X-NewRelic-ID': 'value'}) return 'X-NewRelic-ID' in dict(d.items())
async def start(self, connection, read_until_eof=False): # vk.com return url like this: http://REDIRECT_URI#access_token=... # but aiohttp by default removes all parameters after '#' await super().start(connection, read_until_eof) headers = CIMultiDict(self.headers) location = headers.get(hdrs.LOCATION, None) if location is not None: headers[hdrs.LOCATION] = location.replace('#', '?') self.headers = CIMultiDictProxy(headers) self.raw_headers = tuple(headers.items()) return self
async def start(self, connection, read_until_eof=False): # vk.com return url like this: http://REDIRECT_URI#access_token=... # but aiohttp by default removes all parameters after '#' await super().start(connection, read_until_eof) headers = CIMultiDict(self.headers) location = headers.get(hdrs.LOCATION, None) if location: headers[hdrs.LOCATION] = location.replace('#', '?') self.headers = CIMultiDictProxy(headers) self.raw_headers = tuple(headers.items()) return self
def _prepare_headers(self, headers): """ Add default headers and transform it to CIMultiDict """ # Convert headers to MultiDict result = CIMultiDict(self._default_headers) if headers: if not isinstance(headers, (MultiDictProxy, MultiDict)): headers = CIMultiDict(headers) added_names = set() for key, value in headers.items(): if key in added_names: result.add(key, value) else: result[key] = value added_names.add(key) return result
def _prepare_headers(self, headers): """ Add default headers and transform it to CIMultiDict """ # Convert headers to MultiDict result = CIMultiDict(self._default_headers) if headers: if not isinstance(headers, (MultiDictProxy, MultiDict)): headers = CIMultiDict(headers) added_names = set() for key, value in headers.items(): if key in added_names: result.add(key, value) else: result[key] = value added_names.add(key) return result
def maker(method, path, headers=None, *, version=HttpVersion(1, 1), closing=False, sslcontext=None, secure_proxy_ssl_header=None): if version < HttpVersion(1, 1): # noqa closing = True if headers is None: headers = {} headers = CIMultiDict(headers) app = mock.Mock() app._debug = False app.on_response_prepare = Signal(app) if "HOST" not in headers: headers["HOST"] = "test.local" # noqa message = RawRequestMessage(method, path, version, headers, [(k.encode('utf-8'), v.encode('utf-8')) for k, v in headers.items()], closing, False) payload = mock.Mock() transport = mock.Mock() def get_extra_info(key): # noqa if key == 'sslcontext': return sslcontext else: return None transport.get_extra_info.side_effect = get_extra_info writer = mock.Mock() reader = mock.Mock() req = Request(app, message, payload, transport, reader, writer, secure_proxy_ssl_header=secure_proxy_ssl_header) return req
class ClientRequest: GET_METHODS = {hdrs.METH_GET, hdrs.METH_HEAD, hdrs.METH_OPTIONS} POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} ALL_METHODS = GET_METHODS.union(POST_METHODS).union( {hdrs.METH_DELETE, hdrs.METH_TRACE}) DEFAULT_HEADERS = { hdrs.ACCEPT: '*/*', hdrs.ACCEPT_ENCODING: 'gzip, deflate', } SERVER_SOFTWARE = HttpMessage.SERVER_SOFTWARE body = b'' auth = None response = None response_class = None _writer = None # async task for streaming data _continue = None # waiter future for '100 Continue' response # N.B. # Adding __del__ method with self._writer closing doesn't make sense # because _writer is instance method, thus it keeps a reference to self. # Until writer has finished finalizer will not be called. def __init__(self, method, url, *, params=None, headers=None, skip_auto_headers=frozenset(), data=None, cookies=None, auth=None, encoding='utf-8', version=http.HttpVersion11, compress=None, chunked=None, expect100=False, loop=None, response_class=None, proxy=None, proxy_auth=None, timer=None): if loop is None: loop = asyncio.get_event_loop() assert isinstance(url, URL), url assert isinstance(proxy, (URL, type(None))), proxy if params: q = MultiDict(url.query) url2 = url.with_query(params) q.extend(url2.query) url = url.with_query(q) self.url = url.with_fragment(None) self.original_url = url self.method = method.upper() self.encoding = encoding self.chunked = chunked self.compress = compress self.loop = loop self.response_class = response_class or ClientResponse self._timer = timer if timer is not None else _TimeServiceTimeoutNoop() if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) self.update_version(version) self.update_host(url) self.update_headers(headers) self.update_auto_headers(skip_auto_headers) self.update_cookies(cookies) self.update_content_encoding(data) self.update_auth(auth) self.update_proxy(proxy, proxy_auth) self.update_body_from_data(data, skip_auto_headers) self.update_transfer_encoding() self.update_expect_continue(expect100) @property def host(self): return self.url.host @property def port(self): return self.url.port def update_host(self, url): """Update destination host, port and connection type (ssl).""" # get host/port if not url.host: raise ValueError('Host could not be detected.') # basic auth info username, password = url.user, url.password if username: self.auth = helpers.BasicAuth(username, password or '') # Record entire netloc for usage in host header scheme = url.scheme self.ssl = scheme in ('https', 'wss') def update_version(self, version): """Convert request version to two elements tuple. parser HTTP version '1.1' => (1, 1) """ if isinstance(version, str): v = [l.strip() for l in version.split('.', 1)] try: version = int(v[0]), int(v[1]) except ValueError: raise ValueError( 'Can not parse http version number: {}'.format( version)) from None self.version = version def update_headers(self, headers): """Update request headers.""" self.headers = CIMultiDict() if headers: if isinstance(headers, (dict, MultiDictProxy, MultiDict)): headers = headers.items() for key, value in headers: self.headers.add(key, value) def update_auto_headers(self, skip_auto_headers): self.skip_auto_headers = skip_auto_headers used_headers = set(self.headers) | skip_auto_headers for hdr, val in self.DEFAULT_HEADERS.items(): if hdr not in used_headers: self.headers.add(hdr, val) # add host if hdrs.HOST not in used_headers: netloc = self.url.raw_host if not self.url.is_default_port(): netloc += ':' + str(self.url.port) self.headers[hdrs.HOST] = netloc if hdrs.USER_AGENT not in used_headers: self.headers[hdrs.USER_AGENT] = self.SERVER_SOFTWARE def update_cookies(self, cookies): """Update request cookies header.""" if not cookies: return c = SimpleCookie() if hdrs.COOKIE in self.headers: c.load(self.headers.get(hdrs.COOKIE, '')) del self.headers[hdrs.COOKIE] for name, value in cookies.items(): if isinstance(value, Morsel): # Preserve coded_value mrsl_val = value.get(value.key, Morsel()) mrsl_val.set(value.key, value.value, value.coded_value) c[name] = mrsl_val else: c[name] = value self.headers[hdrs.COOKIE] = c.output(header='', sep=';').strip() def update_content_encoding(self, data): """Set request content encoding.""" if not data: return enc = self.headers.get(hdrs.CONTENT_ENCODING, '').lower() if enc: if self.compress is not False: self.compress = enc # enable chunked, no need to deal with length self.chunked = True elif self.compress: if not isinstance(self.compress, str): self.compress = 'deflate' self.headers[hdrs.CONTENT_ENCODING] = self.compress self.chunked = True # enable chunked, no need to deal with length def update_auth(self, auth): """Set basic auth.""" if auth is None: auth = self.auth if auth is None: return if not isinstance(auth, helpers.BasicAuth): raise TypeError('BasicAuth() tuple is required instead') self.headers[hdrs.AUTHORIZATION] = auth.encode() def update_body_from_data(self, data, skip_auto_headers): if not data: return if isinstance(data, str): data = data.encode(self.encoding) if isinstance(data, (bytes, bytearray)): self.body = data if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' if hdrs.CONTENT_LENGTH not in self.headers and not self.chunked: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) elif isinstance( data, (asyncio.StreamReader, streams.StreamReader, streams.DataQueue)): self.body = data elif asyncio.iscoroutine(data): self.body = data if (hdrs.CONTENT_LENGTH not in self.headers and self.chunked is None): self.chunked = True elif isinstance(data, io.IOBase): assert not isinstance(data, io.StringIO), \ 'attempt to send text data instead of binary' self.body = data if not self.chunked and isinstance(data, io.BytesIO): # Not chunking if content-length can be determined size = len(data.getbuffer()) self.headers[hdrs.CONTENT_LENGTH] = str(size) self.chunked = False elif (not self.chunked and isinstance(data, (io.BufferedReader, io.BufferedRandom))): # Not chunking if content-length can be determined try: size = os.fstat(data.fileno()).st_size - data.tell() self.headers[hdrs.CONTENT_LENGTH] = str(size) self.chunked = False except OSError: # data.fileno() is not supported, e.g. # io.BufferedReader(io.BytesIO(b'data')) self.chunked = True else: self.chunked = True if hasattr(data, 'mode'): if data.mode == 'r': raise ValueError('file {!r} should be open in binary mode' ''.format(data)) if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers and hasattr(data, 'name')): mime = mimetypes.guess_type(data.name)[0] mime = 'application/octet-stream' if mime is None else mime self.headers[hdrs.CONTENT_TYPE] = mime elif isinstance(data, MultipartWriter): self.body = data.serialize() self.headers.update(data.headers) self.chunked = True else: if not isinstance(data, helpers.FormData): data = helpers.FormData(data) self.body = data(self.encoding) if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = data.content_type if data.is_multipart: self.chunked = True else: if (hdrs.CONTENT_LENGTH not in self.headers and not self.chunked): self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_transfer_encoding(self): """Analyze transfer-encoding header.""" te = self.headers.get(hdrs.TRANSFER_ENCODING, '').lower() if self.chunked: if hdrs.CONTENT_LENGTH in self.headers: del self.headers[hdrs.CONTENT_LENGTH] if 'chunked' not in te: self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' else: if 'chunked' in te: self.chunked = True else: self.chunked = None if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_expect_continue(self, expect=False): if expect: self.headers[hdrs.EXPECT] = '100-continue' elif self.headers.get(hdrs.EXPECT, '').lower() == '100-continue': expect = True if expect: self._continue = helpers.create_future(self.loop) def update_proxy(self, proxy, proxy_auth): if proxy and not proxy.scheme == 'http': raise ValueError("Only http proxies are supported") if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): raise ValueError("proxy_auth must be None or BasicAuth() tuple") self.proxy = proxy self.proxy_auth = proxy_auth @asyncio.coroutine def write_bytes(self, request, conn): """Support coroutines that yields bytes objects.""" # 100 response if self._continue is not None: yield from request.drain() yield from self._continue try: if asyncio.iscoroutine(self.body): exc = None value = None stream = self.body while True: try: if exc is not None: result = stream.throw(exc) else: result = stream.send(value) except StopIteration as exc: if isinstance(exc.value, bytes): yield from request.write(exc.value) break except: self.response.close() raise if isinstance(result, asyncio.Future): exc = None value = None try: value = yield result except Exception as err: exc = err elif isinstance(result, (bytes, bytearray)): yield from request.write(result) value = None else: raise ValueError('Bytes object is expected, got: %s.' % type(result)) elif isinstance(self.body, (asyncio.StreamReader, streams.StreamReader)): chunk = yield from self.body.read(streams.DEFAULT_LIMIT) while chunk: yield from request.write(chunk, drain=True) chunk = yield from self.body.read(streams.DEFAULT_LIMIT) elif isinstance(self.body, streams.DataQueue): while True: try: chunk = yield from self.body.read() if not chunk: break yield from request.write(chunk) except streams.EofStream: break elif isinstance(self.body, io.IOBase): chunk = self.body.read(streams.DEFAULT_LIMIT) while chunk: request.write(chunk) chunk = self.body.read(self.chunked) else: if isinstance(self.body, (bytes, bytearray)): self.body = (self.body, ) for chunk in self.body: request.write(chunk) except Exception as exc: new_exc = aiohttp.ClientRequestError( 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc conn.protocol.set_exception(new_exc) else: try: yield from request.write_eof() except Exception as exc: new_exc = aiohttp.ClientRequestError( 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc conn.protocol.set_exception(new_exc) self._writer = None def send(self, conn): # Specify request target: # - CONNECT request must send authority form URI # - not CONNECT proxy must send absolute form URI # - most common is origin form URI if self.method == hdrs.METH_CONNECT: path = '{}:{}'.format(self.url.raw_host, self.url.port) elif self.proxy and not self.ssl: path = str(self.url) else: path = self.url.raw_path if self.url.raw_query_string: path += '?' + self.url.raw_query_string request = http.Request(conn.writer, self.method, path, self.version, loop=self.loop) if self.compress: request.enable_compression(self.compress) if self.chunked is not None: request.enable_chunking() # set default content-type if (self.method in self.POST_METHODS and hdrs.CONTENT_TYPE not in self.skip_auto_headers and hdrs.CONTENT_TYPE not in self.headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' for k, value in self.headers.items(): request.add_header(k, value) request.send_headers() self._writer = helpers.ensure_future(self.write_bytes(request, conn), loop=self.loop) self.response = self.response_class(self.method, self.original_url, writer=self._writer, continue100=self._continue, timer=self._timer) self.response._post_init(self.loop) return self.response @asyncio.coroutine def close(self): if self._writer is not None: try: yield from self._writer finally: self._writer = None def terminate(self): if self._writer is not None: if not self.loop.is_closed(): self._writer.cancel() self._writer = None
class HttpMessage(PayloadWriter): """HttpMessage allows to write headers and payload to a stream.""" HOP_HEADERS = None # Must be set by subclass. SERVER_SOFTWARE = 'Python/{0[0]}.{0[1]} aiohttp/{1}'.format( sys.version_info, aiohttp.__version__) upgrade = False # Connection: UPGRADE websocket = False # Upgrade: WEBSOCKET has_chunked_hdr = False # Transfer-encoding: chunked def __init__(self, transport, version, close, loop=None): super().__init__(transport, loop) self.version = version self.closing = close self.keepalive = None self.length = None self.headers = CIMultiDict() self.headers_sent = False @property def body_length(self): return self.output_length def force_close(self): self.closing = True self.keepalive = False def keep_alive(self): if self.keepalive is None: if self.version < HttpVersion10: # keep alive not supported at all return False if self.version == HttpVersion10: if self.headers.get(hdrs.CONNECTION) == 'keep-alive': return True else: # no headers means we close for Http 1.0 return False else: return not self.closing else: return self.keepalive def is_headers_sent(self): return self.headers_sent def add_header(self, name, value): """Analyze headers. Calculate content length, removes hop headers, etc.""" assert not self.headers_sent, 'headers have been sent already' assert isinstance(name, str), \ 'Header name should be a string, got {!r}'.format(name) assert set(name).issubset(ASCIISET), \ 'Header name should contain ASCII chars, got {!r}'.format(name) assert isinstance(value, str), \ 'Header {!r} should have string value, got {!r}'.format( name, value) name = istr(name) value = value.strip() if name == hdrs.CONTENT_LENGTH: self.length = int(value) if name == hdrs.TRANSFER_ENCODING: self.has_chunked_hdr = value.lower() == 'chunked' if name == hdrs.CONNECTION: val = value.lower() # handle websocket if 'upgrade' in val: self.upgrade = True # connection keep-alive elif 'close' in val: self.keepalive = False elif 'keep-alive' in val: self.keepalive = True elif name == hdrs.UPGRADE: if 'websocket' in value.lower(): self.websocket = True self.headers[name] = value elif name not in self.HOP_HEADERS: # ignore hop-by-hop headers self.headers.add(name, value) def add_headers(self, *headers): """Adds headers to a HTTP message.""" for name, value in headers: self.add_header(name, value) def send_headers(self, _sep=': ', _end='\r\n'): """Writes headers to a stream. Constructs payload writer.""" # Chunked response is only for HTTP/1.1 clients or newer # and there is no Content-Length header is set. # Do not use chunked responses when the response is guaranteed to # not have a response body (304, 204). assert not self.headers_sent, 'headers have been sent already' self.headers_sent = True if not self.chunked and self.autochunked(): self.enable_chunking() if self.chunked: self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' self._add_default_headers() # status + headers headers = self.status_line + ''.join( [k + _sep + v + _end for k, v in self.headers.items()]) headers = headers.encode('utf-8') + b'\r\n' self.buffer_data(headers) def _add_default_headers(self): # set the connection header connection = None if self.upgrade: connection = 'Upgrade' elif not self.closing if self.keepalive is None else self.keepalive: if self.version == HttpVersion10: connection = 'keep-alive' else: if self.version == HttpVersion11: connection = 'close' if connection is not None: self.headers[hdrs.CONNECTION] = connection
async def handleRequest(self, reader, writer): # connection id, for debugging connid = self.nextConnId self.nextConnId += 1 logger.debug(f'{connid}: new incoming connection') # simple HTTP parsing, yes this is a terrible idea. l = await reader.readline() if l == bytes(): raise BadRequest(f'{connid}: unexpected eof') l = l.rstrip(b'\r\n') try: method, rawPath, proto = l.split(b' ') logger.debug(f'{connid}: got {method} {rawPath} {proto}') except ValueError: logger.error(f'{connid}: cannot split line {l}') raise reqUrl = furl(rawPath.decode('utf-8')) headers = CIMultiDict() while True: if len(headers) > 100: raise BadRequest('too many headers') l = await reader.readline() if l == bytes(): raise BadRequest(f'{connid}: unexpected eof in headers') l = l.rstrip(b'\r\n') logger.debug(f'{connid}: got header line {l!r}') # end of headers? if l == bytes(): break try: key, value = l.decode('utf-8').split(':', 1) headers.add(key.strip(), value.strip()) except ValueError: logger.error(f'cannot parse {l}') logger.debug(f'{connid}: {rawPath} {method} got headers {headers}') route = None try: netloc = headers['host'] reqUrl = reqUrl.set(scheme='http', netloc=netloc) logger.debug(f'got request url {reqUrl}') routeKey = None for d in self.domain: m = parse(d, reqUrl.netloc) if m is not None: routeKey = RouteKey(key=m['key'], user=m['user']) logger.debug(f'{connid}: got route key {routeKey}') break route = self.routes[routeKey] except (KeyError, ValueError): logger.info(f'{connid}: cannot find route for {reqUrl}') self.status['noroute'] += 1 # error is written to client later # is this a non-forwarded request? segments = reqUrl.path.segments if len(segments) > 0 and segments[0] == '_conductor': if segments[1] == 'auth': logger.info(f'authorization request for {reqUrl.netloc}') try: nextLoc = reqUrl.query.params['next'].encode('utf-8') except KeyError: nextLoc = b'/' writer.write(b'\r\n'.join([ b'HTTP/1.0 302 Found', b'Location: ' + nextLoc, b'Set-Cookie: authorization=' + segments[2].encode('utf-8') + b'; HttpOnly; Path=/', b'Cache-Control: no-store', b'', b'Follow the white rabbit.' ])) elif segments[1] == 'status': writer.write( b'HTTP/1.0 200 OK\r\nContent-Type: application/json\r\n\r\n' ) self.status['routesTotal'] = len(self.routes) writer.write( json.dumps(self.status, ensure_ascii=True).encode('ascii')) else: writer.write( b'HTTP/1.0 404 Not Found\r\nContent-Type: plain/text\r\n\r\nNot found' ) writer.close() return if not route: writer.write( b'HTTP/1.0 404 Not Found\r\nConnection: close\r\n\r\n') writer.close() return # check authorization cookies = BaseCookie() try: cookies.load(headers['Cookie']) except KeyError: # will be rejected later pass authorized = False for c in cookies.values(): # Only hashed authorization is available to server. if c.key == 'authorization' and self.hashKey( c.value) == route.auth: authorized = True break try: # do not forward auth cookie to the application, so it can’t leak it. del cookies['authorization'] headers['Cookie'] = cookies.output(header='', sep='') except KeyError: # nonexistent cookie is fine pass if not authorized: logger.info( f'{connid}-{reqUrl}: not authorized, cookies sent {cookies.values()}' ) writer.write( b'HTTP/1.0 403 Unauthorized\r\nContent-Type: plain/text\r\nConnection: close\r\n\r\nUnauthorized' ) writer.close() self.status['unauthorized'] += 1 return # try opening the socket try: start = time.time() sockreader, sockwriter = await asyncio.open_unix_connection( path=route.socket) end = time.time() logger.debug(f'opening socket took {end-start}s') except (ConnectionRefusedError, FileNotFoundError, PermissionError): logger.info(f'{connid}-{reqUrl}: route {routeKey} is broken') writer.write( b'HTTP/1.0 502 Bad Gateway\r\nConnection: close\r\n\r\n') writer.close() self.status['broken'] += 1 return # some headers are fixed # not parsing body, so we cannot handle more than one request per connection # XXX: this is super-inefficient if 'Upgrade' not in headers or headers['Upgrade'].lower( ) != 'websocket': headers['Connection'] = 'close' # write http banner plus headers sockwriter.write(method + b' ' + rawPath + b' ' + proto + b'\r\n') for k, v in headers.items(): sockwriter.write(f'{k}: {v}\r\n'.encode('utf-8')) sockwriter.write(b'\r\n') async def beforeABClose(result): if result == 0: # no response received from client logger.info( f'{connid}-{reqUrl}: route {routeKey} got no result from server' ) writer.write( b'HTTP/1.0 502 Bad Gateway\r\nConnection: close\r\n\r\n') await proxy((sockreader, sockwriter, 'sock'), (reader, writer, 'web'), logger=logger, logPrefix=connid, beforeABClose=beforeABClose)
def make_mocked_request(method, path, headers=None, *, version=HttpVersion(1, 1), closing=False, app=None, writer=sentinel, payload_writer=sentinel, protocol=sentinel, transport=sentinel, payload=sentinel, sslcontext=None, secure_proxy_ssl_header=None, client_max_size=1024**2, loop=...): """Creates mocked web.Request testing purposes. Useful in unit tests, when spinning full web server is overkill or specific conditions and errors are hard to trigger. """ task = mock.Mock() if loop is ...: loop = mock.Mock() loop.create_future.return_value = () if version < HttpVersion(1, 1): closing = True if headers: headers = CIMultiDict(headers) raw_hdrs = tuple( (k.encode('utf-8'), v.encode('utf-8')) for k, v in headers.items()) else: headers = CIMultiDict() raw_hdrs = () chunked = 'chunked' in headers.get(hdrs.TRANSFER_ENCODING, '').lower() message = RawRequestMessage( method, path, version, headers, raw_hdrs, closing, False, False, chunked, URL(path)) if app is None: app = _create_app_mock() if protocol is sentinel: protocol = mock.Mock() if transport is sentinel: transport = _create_transport(sslcontext) if writer is sentinel: writer = mock.Mock() writer.transport = transport if payload_writer is sentinel: payload_writer = mock.Mock() payload_writer.write_eof.side_effect = noop payload_writer.drain.side_effect = noop protocol.transport = transport protocol.writer = writer if payload is sentinel: payload = mock.Mock() time_service = mock.Mock() time_service.time.return_value = 12345 time_service.strtime.return_value = "Tue, 15 Nov 1994 08:12:31 GMT" @contextmanager def timeout(*args, **kw): yield time_service.timeout = mock.Mock() time_service.timeout.side_effect = timeout req = Request(message, payload, protocol, payload_writer, time_service, task, loop, secure_proxy_ssl_header=secure_proxy_ssl_header, client_max_size=client_max_size) match_info = UrlMappingMatchInfo({}, mock.Mock()) match_info.add_app(app) req._match_info = match_info return req
class Message: def __init__( self, headers=None, payload=None, from_details=None, to_details=None, contact_details=None, ): if headers: self.headers = headers else: self.headers = CIMultiDict() if from_details: self._from_details = from_details elif 'From' not in self.headers: raise ValueError('From header or from_details is required') if to_details: self._to_details = to_details elif 'To' not in self.headers: raise ValueError('To header or to_details is required') if contact_details: self._contact_details = contact_details self._payload = payload self._raw_payload = None if 'Via' not in self.headers: self.headers['Via'] = 'SIP/2.0/%(protocol)s ' + \ utils.format_host_and_port(self.contact_details['uri']['host'], self.contact_details['uri']['port']) + \ ';branch=%s' % utils.gen_branch(10) @property def auth(self): if not hasattr(self, '_auth'): self._auth = Auth.from_message(self) return self._auth @property def payload(self): if self._payload: return self._payload elif self._raw_payload: self._payload = self._raw_payload.decode() return self._payload else: return '' @payload.setter def payload(self, payload): self._payload = payload @property def from_details(self): if not hasattr(self, '_from_details'): self._from_details = Contact.from_header(self.headers['From']) return self._from_details @from_details.setter def from_details(self, from_details): self._from_details = from_details @property def to_details(self): if not hasattr(self, '_to_details'): self._to_details = Contact.from_header(self.headers['To']) return self._to_details @to_details.setter def to_details(self, to_details): self._to_details = to_details @property def contact_details(self): if not hasattr(self, '_contact_details'): if 'Contact' in self.headers: self._contact_details = Contact.from_header( self.headers['Contact']) else: self._contact_details = None return self._contact_details @contact_details.setter def contact_details(self, contact_details): self._contact_details = contact_details @property def content_type(self): return self.headers['Content-Type'] @content_type.setter def content_type(self, content_type): self.headers['Content-Type'] = content_type @property def cseq(self): if not hasattr(self, '_cseq'): self._cseq = int(self.headers['CSeq'].split(' ')[0]) return self._cseq @cseq.setter def cseq(self, cseq): self._cseq = int(cseq) @property def method(self): if not hasattr(self, '_method'): self._method = self.headers['CSeq'].split(' ')[1] return self._method @method.setter def method(self, method): self._method = method def __str__(self): if self._payload: self._raw_payload = self._payload.encode() elif not self._raw_payload: self._raw_payload = b'' msg = self._make_headers() return msg + self.payload def encode(self, *args, **kwargs): if self._payload: self._raw_payload = self._payload.encode(*args, **kwargs) elif not self._raw_payload: self._raw_payload = b'' msg = self._make_headers() return msg.encode(*args, **kwargs) + self._raw_payload def _make_headers(self): if hasattr(self, '_from_details'): self.headers['From'] = str(self.from_details) if hasattr(self, '_to_details'): self.headers['To'] = str(self.to_details) if hasattr(self, '_contact_details') and 'Contact' not in self.headers: self.headers['Contact'] = str(self.contact_details) if hasattr(self, '_cseq'): self.headers['CSeq'] = '%s %s' % (self.cseq, self.method) elif hasattr(self, '_method'): self.headers['CSeq'] = '%s %s' % (self.cseq, self.method) self.headers['Content-Length'] = str(len(self._raw_payload)) if 'Max-Forwards' not in self.headers: self.headers['Max-Forwards'] = '70' if 'Call-ID' not in self.headers: self.headers['Call-ID'] = uuid.uuid4() return self._format_headers() def _format_headers(self): msg = [] for k, v in sorted(self.headers.items()): if k == 'Via': if isinstance(v, (list, tuple)): msg = ['%s: %s' % (k, i) for i in v] + msg else: msg.insert(0, '%s: %s' % (k, v)) else: if isinstance(v, (list, tuple)): msg.extend(['%s: %s' % (k, i) for i in v]) else: msg.append('%s: %s' % (k, v)) msg.append(utils.EOL) return utils.EOL.join(msg) def parsed_xml(self): if 'Content-Type' not in self.headers: return None if not self.headers['Content-Type'].endswith('+xml'): return None return PyQuery(self.payload).remove_namespaces() @classmethod def from_raw_headers(cls, raw_headers): headers = CIMultiDict() decoded_headers = raw_headers.decode().split(utils.EOL) for line in decoded_headers[1:]: k, v = line.split(': ', 1) if k in headers: o = headers.setdefault(k, []) if not isinstance(o, list): o = [o] o.append(v) headers[k] = o else: headers[k] = v m = FIRST_LINE_PATTERN['response']['regex'].match(decoded_headers[0]) if m: d = m.groupdict() return Response(status_code=int(d['status_code']), status_message=d['status_message'], headers=headers, first_line=decoded_headers[0]) else: m = FIRST_LINE_PATTERN['request']['regex'].match( decoded_headers[0]) if m: d = m.groupdict() cseq, _ = headers['CSeq'].split() return Request(method=d['method'], headers=headers, cseq=int(cseq), first_line=decoded_headers[0]) else: LOG.debug(decoded_headers) raise ValueError('Not a SIP message')
class HttpMessage(ABC): """HttpMessage allows to write headers and payload to a stream. For example, lets say we want to read file then compress it with deflate compression and then send it with chunked transfer encoding, code may look like this: >>> response = aiohttp.Response(transport, 200) We have to use deflate compression first: >>> response.add_compression_filter('deflate') Then we want to split output stream into chunks of 1024 bytes size: >>> response.add_chunking_filter(1024) We can add headers to response with add_headers() method. add_headers() does not send data to transport, send_headers() sends request/response line and then sends headers: >>> response.add_headers( ... ('Content-Disposition', 'attachment; filename="..."')) >>> response.send_headers() Now we can use chunked writer to write stream to a network stream. First call to write() method sends response status line and headers, add_header() and add_headers() method unavailable at this stage: >>> with open('...', 'rb') as f: ... chunk = fp.read(8192) ... while chunk: ... response.write(chunk) ... chunk = fp.read(8192) >>> response.write_eof() """ writer = None # 'filter' is being used for altering write() behaviour, # add_chunking_filter adds deflate/gzip compression and # add_compression_filter splits incoming data into a chunks. filter = None HOP_HEADERS = None # Must be set by subclass. SERVER_SOFTWARE = 'Python/{0[0]}.{0[1]} aiohttp/{1}'.format( sys.version_info, aiohttp.__version__) upgrade = False # Connection: UPGRADE websocket = False # Upgrade: WEBSOCKET has_chunked_hdr = False # Transfer-encoding: chunked # subclass can enable auto sending headers with write() call, # this is useful for wsgi's start_response implementation. _send_headers = False def __init__(self, transport, version, close): self.transport = transport self._version = version self.closing = close self.keepalive = None self.chunked = False self.length = None self.headers = CIMultiDict() self.headers_sent = False self.output_length = 0 self.headers_length = 0 self._output_size = 0 @property @abstractmethod def status_line(self): return b'' @abstractmethod def autochunked(self): return False @property def version(self): return self._version @property def body_length(self): return self.output_length - self.headers_length def force_close(self): self.closing = True self.keepalive = False def enable_chunked_encoding(self): self.chunked = True def keep_alive(self): if self.keepalive is None: if self.version < HttpVersion10: # keep alive not supported at all return False if self.version == HttpVersion10: if self.headers.get(hdrs.CONNECTION) == 'keep-alive': return True else: # no headers means we close for Http 1.0 return False else: return not self.closing else: return self.keepalive def is_headers_sent(self): return self.headers_sent def add_header(self, name, value): """Analyze headers. Calculate content length, removes hop headers, etc.""" assert not self.headers_sent, 'headers have been sent already' assert isinstance(name, str), \ 'Header name should be a string, got {!r}'.format(name) assert set(name).issubset(ASCIISET), \ 'Header name should contain ASCII chars, got {!r}'.format(name) assert isinstance(value, str), \ 'Header {!r} should have string value, got {!r}'.format( name, value) name = upstr(name) value = value.strip() if name == hdrs.CONTENT_LENGTH: self.length = int(value) if name == hdrs.TRANSFER_ENCODING: self.has_chunked_hdr = value.lower().strip() == 'chunked' if name == hdrs.CONNECTION: val = value.lower() # handle websocket if 'upgrade' in val: self.upgrade = True # connection keep-alive elif 'close' in val: self.keepalive = False elif 'keep-alive' in val: self.keepalive = True elif name == hdrs.UPGRADE: if 'websocket' in value.lower(): self.websocket = True self.headers[name] = value elif name not in self.HOP_HEADERS: # ignore hop-by-hop headers self.headers.add(name, value) def add_headers(self, *headers): """Adds headers to a HTTP message.""" for name, value in headers: self.add_header(name, value) def send_headers(self, _sep=': ', _end='\r\n'): """Writes headers to a stream. Constructs payload writer.""" # Chunked response is only for HTTP/1.1 clients or newer # and there is no Content-Length header is set. # Do not use chunked responses when the response is guaranteed to # not have a response body (304, 204). assert not self.headers_sent, 'headers have been sent already' self.headers_sent = True if self.chunked or self.autochunked(): self.writer = self._write_chunked_payload() self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' elif self.length is not None: self.writer = self._write_length_payload(self.length) else: self.writer = self._write_eof_payload() next(self.writer) self._add_default_headers() # status + headers headers = self.status_line + ''.join( [k + _sep + v + _end for k, v in self.headers.items()]) headers = headers.encode('utf-8') + b'\r\n' self.output_length += len(headers) self.headers_length = len(headers) self.transport.write(headers) def _add_default_headers(self): # set the connection header connection = None if self.upgrade: connection = 'upgrade' elif not self.closing if self.keepalive is None else self.keepalive: if self.version == HttpVersion10: connection = 'keep-alive' else: if self.version == HttpVersion11: connection = 'close' if connection is not None: self.headers[hdrs.CONNECTION] = connection def write(self, chunk, *, drain=False, EOF_MARKER=EOF_MARKER, EOL_MARKER=EOL_MARKER): """Writes chunk of data to a stream by using different writers. writer uses filter to modify chunk of data. write_eof() indicates end of stream. writer can't be used after write_eof() method being called. write() return drain future. """ assert (isinstance(chunk, (bytes, bytearray)) or chunk is EOF_MARKER), chunk size = self.output_length if self._send_headers and not self.headers_sent: self.send_headers() assert self.writer is not None, 'send_headers() is not called.' if self.filter: chunk = self.filter.send(chunk) while chunk not in (EOF_MARKER, EOL_MARKER): if chunk: self.writer.send(chunk) chunk = next(self.filter) else: if chunk is not EOF_MARKER: self.writer.send(chunk) self._output_size += self.output_length - size if self._output_size > 64 * 1024: if drain: self._output_size = 0 return self.transport.drain() return () def write_eof(self): self.write(EOF_MARKER) try: self.writer.throw(aiohttp.EofStream()) except StopIteration: pass return self.transport.drain() def _write_chunked_payload(self): """Write data in chunked transfer encoding.""" while True: try: chunk = yield except aiohttp.EofStream: self.transport.write(b'0\r\n\r\n') self.output_length += 5 break chunk = bytes(chunk) chunk_len = '{:x}\r\n'.format(len(chunk)).encode('ascii') self.transport.write(chunk_len + chunk + b'\r\n') self.output_length += len(chunk_len) + len(chunk) + 2 def _write_length_payload(self, length): """Write specified number of bytes to a stream.""" while True: try: chunk = yield except aiohttp.EofStream: break if length: l = len(chunk) if length >= l: self.transport.write(chunk) self.output_length += l length = length - l else: self.transport.write(chunk[:length]) self.output_length += length length = 0 def _write_eof_payload(self): while True: try: chunk = yield except aiohttp.EofStream: break self.transport.write(chunk) self.output_length += len(chunk) @wrap_payload_filter def add_chunking_filter(self, chunk_size=16 * 1024, *, EOF_MARKER=EOF_MARKER, EOL_MARKER=EOL_MARKER): """Split incoming stream into chunks.""" buf = bytearray() chunk = yield while True: if chunk is EOF_MARKER: if buf: yield buf yield EOF_MARKER else: buf.extend(chunk) while len(buf) >= chunk_size: chunk = bytes(buf[:chunk_size]) del buf[:chunk_size] yield chunk chunk = yield EOL_MARKER @wrap_payload_filter def add_compression_filter(self, encoding='deflate', *, EOF_MARKER=EOF_MARKER, EOL_MARKER=EOL_MARKER): """Compress incoming stream with deflate or gzip encoding.""" zlib_mode = (16 + zlib.MAX_WBITS if encoding == 'gzip' else -zlib.MAX_WBITS) zcomp = zlib.compressobj(wbits=zlib_mode) chunk = yield while True: if chunk is EOF_MARKER: yield zcomp.flush() chunk = yield EOF_MARKER else: yield zcomp.compress(chunk) chunk = yield EOL_MARKER
class ClientRequest: GET_METHODS = {hdrs.METH_GET, hdrs.METH_HEAD, hdrs.METH_OPTIONS} POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} ALL_METHODS = GET_METHODS.union(POST_METHODS).union({hdrs.METH_DELETE, hdrs.METH_TRACE}) DEFAULT_HEADERS = {hdrs.ACCEPT: "*/*", hdrs.ACCEPT_ENCODING: "gzip, deflate"} SERVER_SOFTWARE = HttpMessage.SERVER_SOFTWARE body = b"" auth = None response = None response_class = None _writer = None # async task for streaming data _continue = None # waiter future for '100 Continue' response # N.B. # Adding __del__ method with self._writer closing doesn't make sense # because _writer is instance method, thus it keeps a reference to self. # Until writer has finished finalizer will not be called. def __init__( self, method, url, *, params=None, headers=None, skip_auto_headers=frozenset(), data=None, cookies=None, auth=None, encoding="utf-8", version=aiohttp.HttpVersion11, compress=None, chunked=None, expect100=False, loop=None, response_class=None ): if loop is None: loop = asyncio.get_event_loop() self.url = url self.method = method.upper() self.encoding = encoding self.chunked = chunked self.compress = compress self.loop = loop self.response_class = response_class or ClientResponse if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) self.update_version(version) self.update_host(url) self.update_path(params) self.update_headers(headers) self.update_auto_headers(skip_auto_headers) self.update_cookies(cookies) self.update_content_encoding() self.update_auth(auth) self.update_body_from_data(data, skip_auto_headers) self.update_transfer_encoding() self.update_expect_continue(expect100) def update_host(self, url): """Update destination host, port and connection type (ssl).""" url_parsed = urllib.parse.urlsplit(url) # check for network location part netloc = url_parsed.netloc if not netloc: raise ValueError("Host could not be detected.") # get host/port host = url_parsed.hostname if not host: raise ValueError("Host could not be detected.") try: port = url_parsed.port except ValueError: raise ValueError("Port number could not be converted.") from None # check domain idna encoding try: netloc = netloc.encode("idna").decode("utf-8") host = host.encode("idna").decode("utf-8") except UnicodeError: raise ValueError("URL has an invalid label.") # basic auth info username, password = url_parsed.username, url_parsed.password if username: self.auth = helpers.BasicAuth(username, password or "") netloc = netloc.split("@", 1)[1] # Record entire netloc for usage in host header self.netloc = netloc scheme = url_parsed.scheme self.ssl = scheme in ("https", "wss") # set port number if it isn't already set if not port: if self.ssl: port = HTTPS_PORT else: port = HTTP_PORT self.host, self.port, self.scheme = host, port, scheme def update_version(self, version): """Convert request version to two elements tuple. parser HTTP version '1.1' => (1, 1) """ if isinstance(version, str): v = [l.strip() for l in version.split(".", 1)] try: version = int(v[0]), int(v[1]) except ValueError: raise ValueError("Can not parse http version number: {}".format(version)) from None self.version = version def update_path(self, params): """Build path.""" # extract path scheme, netloc, path, query, fragment = urllib.parse.urlsplit(self.url) if not path: path = "/" if isinstance(params, collections.Mapping): params = list(params.items()) if params: if not isinstance(params, str): params = urllib.parse.urlencode(params) if query: query = "%s&%s" % (query, params) else: query = params self.path = urllib.parse.urlunsplit(("", "", helpers.requote_uri(path), query, fragment)) self.url = urllib.parse.urlunsplit((scheme, netloc, self.path, "", "")) def update_headers(self, headers): """Update request headers.""" self.headers = CIMultiDict() if headers: if isinstance(headers, dict): headers = headers.items() elif isinstance(headers, (MultiDictProxy, MultiDict)): headers = headers.items() for key, value in headers: self.headers.add(key, value) def update_auto_headers(self, skip_auto_headers): self.skip_auto_headers = skip_auto_headers used_headers = set(self.headers) | skip_auto_headers for hdr, val in self.DEFAULT_HEADERS.items(): if hdr not in used_headers: self.headers.add(hdr, val) # add host if hdrs.HOST not in used_headers: self.headers[hdrs.HOST] = self.netloc if hdrs.USER_AGENT not in used_headers: self.headers[hdrs.USER_AGENT] = self.SERVER_SOFTWARE def update_cookies(self, cookies): """Update request cookies header.""" if not cookies: return c = http.cookies.SimpleCookie() if hdrs.COOKIE in self.headers: c.load(self.headers.get(hdrs.COOKIE, "")) del self.headers[hdrs.COOKIE] if isinstance(cookies, dict): cookies = cookies.items() for name, value in cookies: if isinstance(value, http.cookies.Morsel): c[value.key] = value.value else: c[name] = value self.headers[hdrs.COOKIE] = c.output(header="", sep=";").strip() def update_content_encoding(self): """Set request content encoding.""" enc = self.headers.get(hdrs.CONTENT_ENCODING, "").lower() if enc: if self.compress is not False: self.compress = enc # enable chunked, no need to deal with length self.chunked = True elif self.compress: if not isinstance(self.compress, str): self.compress = "deflate" self.headers[hdrs.CONTENT_ENCODING] = self.compress self.chunked = True # enable chunked, no need to deal with length def update_auth(self, auth): """Set basic auth.""" if auth is None: auth = self.auth if auth is None: return if not isinstance(auth, helpers.BasicAuth): warnings.warn("BasicAuth() tuple is required instead ", DeprecationWarning) auth = helpers.BasicAuth(*auth) self.headers[hdrs.AUTHORIZATION] = auth.encode() def update_body_from_data(self, data, skip_auto_headers): if not data: return if isinstance(data, str): data = data.encode(self.encoding) if isinstance(data, (bytes, bytearray)): self.body = data if hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers: self.headers[hdrs.CONTENT_TYPE] = "application/octet-stream" if hdrs.CONTENT_LENGTH not in self.headers and not self.chunked: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) elif isinstance(data, (asyncio.StreamReader, streams.DataQueue)): self.body = data elif asyncio.iscoroutine(data): self.body = data if hdrs.CONTENT_LENGTH not in self.headers and self.chunked is None: self.chunked = True elif isinstance(data, io.IOBase): assert not isinstance(data, io.StringIO), "attempt to send text data instead of binary" self.body = data if not self.chunked and isinstance(data, io.BytesIO): # Not chunking if content-length can be determined size = len(data.getbuffer()) self.headers[hdrs.CONTENT_LENGTH] = str(size) self.chunked = False elif not self.chunked and isinstance(data, io.BufferedReader): # Not chunking if content-length can be determined try: size = os.fstat(data.fileno()).st_size - data.tell() self.headers[hdrs.CONTENT_LENGTH] = str(size) self.chunked = False except OSError: # data.fileno() is not supported, e.g. # io.BufferedReader(io.BytesIO(b'data')) self.chunked = True else: self.chunked = True if hasattr(data, "mode"): if data.mode == "r": raise ValueError("file {!r} should be open in binary mode" "".format(data)) if ( hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers and hasattr(data, "name") ): mime = mimetypes.guess_type(data.name)[0] mime = "application/octet-stream" if mime is None else mime self.headers[hdrs.CONTENT_TYPE] = mime elif isinstance(data, MultipartWriter): self.body = data.serialize() self.headers.update(data.headers) self.chunked = self.chunked or 8192 else: if not isinstance(data, helpers.FormData): data = helpers.FormData(data) self.body = data(self.encoding) if hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers: self.headers[hdrs.CONTENT_TYPE] = data.content_type if data.is_multipart: self.chunked = self.chunked or 8192 else: if hdrs.CONTENT_LENGTH not in self.headers and not self.chunked: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_transfer_encoding(self): """Analyze transfer-encoding header.""" te = self.headers.get(hdrs.TRANSFER_ENCODING, "").lower() if self.chunked: if hdrs.CONTENT_LENGTH in self.headers: del self.headers[hdrs.CONTENT_LENGTH] if "chunked" not in te: self.headers[hdrs.TRANSFER_ENCODING] = "chunked" self.chunked = self.chunked if type(self.chunked) is int else 8192 else: if "chunked" in te: self.chunked = 8192 else: self.chunked = None if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_expect_continue(self, expect=False): if expect: self.headers[hdrs.EXPECT] = "100-continue" elif self.headers.get(hdrs.EXPECT, "").lower() == "100-continue": expect = True if expect: self._continue = asyncio.Future(loop=self.loop) @asyncio.coroutine def write_bytes(self, request, reader): """Support coroutines that yields bytes objects.""" # 100 response if self._continue is not None: yield from self._continue try: if asyncio.iscoroutine(self.body): request.transport.set_tcp_nodelay(True) exc = None value = None stream = self.body while True: try: if exc is not None: result = stream.throw(exc) else: result = stream.send(value) except StopIteration as exc: if isinstance(exc.value, bytes): yield from request.write(exc.value, drain=True) break except: self.response.close() raise if isinstance(result, asyncio.Future): exc = None value = None try: value = yield result except Exception as err: exc = err elif isinstance(result, (bytes, bytearray)): yield from request.write(result, drain=True) value = None else: raise ValueError("Bytes object is expected, got: %s." % type(result)) elif isinstance(self.body, asyncio.StreamReader): request.transport.set_tcp_nodelay(True) chunk = yield from self.body.read(streams.DEFAULT_LIMIT) while chunk: yield from request.write(chunk, drain=True) chunk = yield from self.body.read(streams.DEFAULT_LIMIT) elif isinstance(self.body, streams.DataQueue): request.transport.set_tcp_nodelay(True) while True: try: chunk = yield from self.body.read() if chunk is EOF_MARKER: break yield from request.write(chunk, drain=True) except streams.EofStream: break elif isinstance(self.body, io.IOBase): chunk = self.body.read(self.chunked) while chunk: request.write(chunk) chunk = self.body.read(self.chunked) request.transport.set_tcp_nodelay(True) else: if isinstance(self.body, (bytes, bytearray)): self.body = (self.body,) for chunk in self.body: request.write(chunk) request.transport.set_tcp_nodelay(True) except Exception as exc: new_exc = aiohttp.ClientRequestError("Can not write request body for %s" % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc reader.set_exception(new_exc) else: assert request.transport.tcp_nodelay try: ret = request.write_eof() # NB: in asyncio 3.4.1+ StreamWriter.drain() is coroutine # see bug #170 if asyncio.iscoroutine(ret) or isinstance(ret, asyncio.Future): yield from ret except Exception as exc: new_exc = aiohttp.ClientRequestError("Can not write request body for %s" % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc reader.set_exception(new_exc) self._writer = None def send(self, writer, reader): writer.set_tcp_cork(True) request = aiohttp.Request(writer, self.method, self.path, self.version) if self.compress: request.add_compression_filter(self.compress) if self.chunked is not None: request.enable_chunked_encoding() request.add_chunking_filter(self.chunked) # set default content-type if ( self.method in self.POST_METHODS and hdrs.CONTENT_TYPE not in self.skip_auto_headers and hdrs.CONTENT_TYPE not in self.headers ): self.headers[hdrs.CONTENT_TYPE] = "application/octet-stream" for k, value in self.headers.items(): request.add_header(k, value) request.send_headers() self._writer = helpers.ensure_future(self.write_bytes(request, reader), loop=self.loop) self.response = self.response_class( self.method, self.url, self.host, writer=self._writer, continue100=self._continue ) self.response._post_init(self.loop) return self.response @asyncio.coroutine def close(self): if self._writer is not None: try: yield from self._writer finally: self._writer = None def terminate(self): if self._writer is not None: if hasattr(self.loop, "is_closed"): if not self.loop.is_closed(): self._writer.cancel() else: self._writer.cancel() self._writer = None
class BaseResponse: __slots__ = ("body", "status", "content_type", "headers", "_cookies", "_protocol") charset = 'utf-8' max_cookie_size = 4093 def __init__(self, body=None, status=200, headers=None, content_type="text/plain"): self.content_type = content_type self.body = self._encode_body(body) self.status = status self.headers = CIMultiDict(headers or {}) self._cookies = None self._protocol = None def set_protocol(self, protocol): self._protocol = protocol def has_protocol(self): return self._protocol is not None @staticmethod def _encode_body(data): try: if not isinstance(data, bytes): return data.encode() return data except AttributeError: return str(data or "").encode() def _parse_headers(self): headers = b"" for name, value in self.headers.items(): try: headers += b"%b: %b\r\n" % ( name.encode(), value.encode(self.charset), ) except AttributeError: headers += b"%b: %b\r\n" % ( str(name).encode(), str(value).encode(self.charset), ) return headers def set_cookie(self, key, value="", max_age=None, expires=None, path="/", domain=None, secure=False, httponly=False, samesite=None): cookie = http.cookies.SimpleCookie() cookie[key] = value if max_age is not None: cookie[key]["max-age"] = max_age # type: ignore if expires is not None: cookie[key]["expires"] = expires # type: ignore if path is not None: cookie[key]["path"] = path if domain is not None: cookie[key]["domain"] = domain if secure: cookie[key]["secure"] = True if httponly: cookie[key]["httponly"] = True if samesite: cookie[key]["samesite"] = samesite cookie_val = cookie.output(header="").strip() self.headers.add('Set-Cookie', cookie_val) def delete_cookie(self, key, path="/", domain=None): self.set_cookie(key, expires=0, max_age=0, path=path, domain=domain) def get_headers(self, version="1.1", keep_alive=False, keep_alive_timeout=None): timeout_header = b"" if keep_alive and keep_alive_timeout is not None: timeout_header = b"Keep-Alive: %d\r\n" % keep_alive_timeout self.headers["Content-Type"] = self.headers.get( "Content-Type", self.content_type) if self.status in (304, 412): self.headers = remove_entity_headers(self.headers) headers = self._parse_headers() if self.status is 200: description = b"OK" else: description = STATUS_TEXT.get(self.status, b"UNKNOWN RESPONSE") return (b"HTTP/%b %d %b\r\n" b"Connection: %b\r\n" b"%b" b"%b\r\n") % (version.encode(), self.status, description, b"keep-alive" if keep_alive else b"close", timeout_header, headers) async def output(self, version="1.1", keep_alive=False, keep_alive_timeout=None): if has_message_body(self.status): body = self.body self.headers["Content-Length"] = self.headers.get( "Content-Length", len(self.body)) else: body = b"" return self.get_headers(version, keep_alive, keep_alive_timeout) + b"%b" % body
class ClientRequest: GET_METHODS = {hdrs.METH_GET, hdrs.METH_HEAD, hdrs.METH_OPTIONS} POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} ALL_METHODS = GET_METHODS.union(POST_METHODS).union( {hdrs.METH_DELETE, hdrs.METH_TRACE}) DEFAULT_HEADERS = { hdrs.ACCEPT: '*/*', hdrs.ACCEPT_ENCODING: 'gzip, deflate', } SERVER_SOFTWARE = HttpMessage.SERVER_SOFTWARE body = b'' auth = None response = None response_class = None _writer = None # async task for streaming data _continue = None # waiter future for '100 Continue' response # N.B. # Adding __del__ method with self._writer closing doesn't make sense # because _writer is instance method, thus it keeps a reference to self. # Until writer has finished finalizer will not be called. def __init__(self, method, url, *, params=None, headers=None, skip_auto_headers=frozenset(), data=None, cookies=None, auth=None, encoding='utf-8', version=aiohttp.HttpVersion11, compress=None, chunked=None, expect100=False, loop=None, response_class=None, proxy=None, proxy_auth=None, timeout=5*60): if loop is None: loop = asyncio.get_event_loop() assert isinstance(url, URL), url assert isinstance(proxy, (URL, type(None))), proxy if params: q = MultiDict(url.query) url2 = url.with_query(params) q.extend(url2.query) url = url.with_query(q) self.url = url.with_fragment(None) self.method = method.upper() self.encoding = encoding self.chunked = chunked self.compress = compress self.loop = loop self.response_class = response_class or ClientResponse self._timeout = timeout if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) self.update_version(version) self.update_host(url) self.update_headers(headers) self.update_auto_headers(skip_auto_headers) self.update_cookies(cookies) self.update_content_encoding(data) self.update_auth(auth) self.update_proxy(proxy, proxy_auth) self.update_body_from_data(data, skip_auto_headers) self.update_transfer_encoding() self.update_expect_continue(expect100) @property def host(self): return self.url.host @property def port(self): return self.url.port def update_host(self, url): """Update destination host, port and connection type (ssl).""" # get host/port if not url.host: raise ValueError('Host could not be detected.') # basic auth info username, password = url.user, url.password if username: self.auth = helpers.BasicAuth(username, password or '') # Record entire netloc for usage in host header scheme = url.scheme self.ssl = scheme in ('https', 'wss') def update_version(self, version): """Convert request version to two elements tuple. parser HTTP version '1.1' => (1, 1) """ if isinstance(version, str): v = [l.strip() for l in version.split('.', 1)] try: version = int(v[0]), int(v[1]) except ValueError: raise ValueError( 'Can not parse http version number: {}' .format(version)) from None self.version = version def update_headers(self, headers): """Update request headers.""" self.headers = CIMultiDict() if headers: if isinstance(headers, dict): headers = headers.items() elif isinstance(headers, (MultiDictProxy, MultiDict)): headers = headers.items() for key, value in headers: self.headers.add(key, value) def update_auto_headers(self, skip_auto_headers): self.skip_auto_headers = skip_auto_headers used_headers = set(self.headers) | skip_auto_headers for hdr, val in self.DEFAULT_HEADERS.items(): if hdr not in used_headers: self.headers.add(hdr, val) # add host if hdrs.HOST not in used_headers: netloc = self.url.host if not self.url.is_default_port(): netloc += ':' + str(self.url.port) self.headers[hdrs.HOST] = netloc if hdrs.USER_AGENT not in used_headers: self.headers[hdrs.USER_AGENT] = self.SERVER_SOFTWARE def update_cookies(self, cookies): """Update request cookies header.""" if not cookies: return c = http.cookies.SimpleCookie() if hdrs.COOKIE in self.headers: c.load(self.headers.get(hdrs.COOKIE, '')) del self.headers[hdrs.COOKIE] for name, value in cookies.items(): if isinstance(value, http.cookies.Morsel): c[value.key] = value.value else: c[name] = value self.headers[hdrs.COOKIE] = c.output(header='', sep=';').strip() def update_content_encoding(self, data): """Set request content encoding.""" if not data: return enc = self.headers.get(hdrs.CONTENT_ENCODING, '').lower() if enc: if self.compress is not False: self.compress = enc # enable chunked, no need to deal with length self.chunked = True elif self.compress: if not isinstance(self.compress, str): self.compress = 'deflate' self.headers[hdrs.CONTENT_ENCODING] = self.compress self.chunked = True # enable chunked, no need to deal with length def update_auth(self, auth): """Set basic auth.""" if auth is None: auth = self.auth if auth is None: return if not isinstance(auth, helpers.BasicAuth): raise TypeError('BasicAuth() tuple is required instead') self.headers[hdrs.AUTHORIZATION] = auth.encode() def update_body_from_data(self, data, skip_auto_headers): if not data: return if isinstance(data, str): data = data.encode(self.encoding) if isinstance(data, (bytes, bytearray)): self.body = data if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' if hdrs.CONTENT_LENGTH not in self.headers and not self.chunked: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) elif isinstance(data, (asyncio.StreamReader, streams.StreamReader, streams.DataQueue)): self.body = data elif asyncio.iscoroutine(data): self.body = data if (hdrs.CONTENT_LENGTH not in self.headers and self.chunked is None): self.chunked = True elif isinstance(data, io.IOBase): assert not isinstance(data, io.StringIO), \ 'attempt to send text data instead of binary' self.body = data if not self.chunked and isinstance(data, io.BytesIO): # Not chunking if content-length can be determined size = len(data.getbuffer()) self.headers[hdrs.CONTENT_LENGTH] = str(size) self.chunked = False elif (not self.chunked and isinstance(data, (io.BufferedReader, io.BufferedRandom))): # Not chunking if content-length can be determined try: size = os.fstat(data.fileno()).st_size - data.tell() self.headers[hdrs.CONTENT_LENGTH] = str(size) self.chunked = False except OSError: # data.fileno() is not supported, e.g. # io.BufferedReader(io.BytesIO(b'data')) self.chunked = True else: self.chunked = True if hasattr(data, 'mode'): if data.mode == 'r': raise ValueError('file {!r} should be open in binary mode' ''.format(data)) if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers and hasattr(data, 'name')): mime = mimetypes.guess_type(data.name)[0] mime = 'application/octet-stream' if mime is None else mime self.headers[hdrs.CONTENT_TYPE] = mime elif isinstance(data, MultipartWriter): self.body = data.serialize() self.headers.update(data.headers) self.chunked = self.chunked or 8192 else: if not isinstance(data, helpers.FormData): data = helpers.FormData(data) self.body = data(self.encoding) if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = data.content_type if data.is_multipart: self.chunked = self.chunked or 8192 else: if (hdrs.CONTENT_LENGTH not in self.headers and not self.chunked): self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_transfer_encoding(self): """Analyze transfer-encoding header.""" te = self.headers.get(hdrs.TRANSFER_ENCODING, '').lower() if self.chunked: if hdrs.CONTENT_LENGTH in self.headers: del self.headers[hdrs.CONTENT_LENGTH] if 'chunked' not in te: self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' self.chunked = self.chunked if type(self.chunked) is int else 8192 else: if 'chunked' in te: self.chunked = 8192 else: self.chunked = None if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_expect_continue(self, expect=False): if expect: self.headers[hdrs.EXPECT] = '100-continue' elif self.headers.get(hdrs.EXPECT, '').lower() == '100-continue': expect = True if expect: self._continue = helpers.create_future(self.loop) def update_proxy(self, proxy, proxy_auth): if proxy and not proxy.scheme == 'http': raise ValueError("Only http proxies are supported") if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): raise ValueError("proxy_auth must be None or BasicAuth() tuple") self.proxy = proxy self.proxy_auth = proxy_auth @asyncio.coroutine def write_bytes(self, request, reader): """Support coroutines that yields bytes objects.""" # 100 response if self._continue is not None: yield from self._continue try: if asyncio.iscoroutine(self.body): request.transport.set_tcp_nodelay(True) exc = None value = None stream = self.body while True: try: if exc is not None: result = stream.throw(exc) else: result = stream.send(value) except StopIteration as exc: if isinstance(exc.value, bytes): yield from request.write(exc.value, drain=True) break except: self.response.close() raise if isinstance(result, asyncio.Future): exc = None value = None try: value = yield result except Exception as err: exc = err elif isinstance(result, (bytes, bytearray)): yield from request.write(result, drain=True) value = None else: raise ValueError( 'Bytes object is expected, got: %s.' % type(result)) elif isinstance(self.body, (asyncio.StreamReader, streams.StreamReader)): request.transport.set_tcp_nodelay(True) chunk = yield from self.body.read(streams.DEFAULT_LIMIT) while chunk: yield from request.write(chunk, drain=True) chunk = yield from self.body.read(streams.DEFAULT_LIMIT) elif isinstance(self.body, streams.DataQueue): request.transport.set_tcp_nodelay(True) while True: try: chunk = yield from self.body.read() if chunk is EOF_MARKER: break yield from request.write(chunk, drain=True) except streams.EofStream: break elif isinstance(self.body, io.IOBase): chunk = self.body.read(self.chunked) while chunk: request.write(chunk) chunk = self.body.read(self.chunked) request.transport.set_tcp_nodelay(True) else: if isinstance(self.body, (bytes, bytearray)): self.body = (self.body,) for chunk in self.body: request.write(chunk) request.transport.set_tcp_nodelay(True) except Exception as exc: new_exc = aiohttp.ClientRequestError( 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc reader.set_exception(new_exc) else: assert request.transport.tcp_nodelay try: ret = request.write_eof() # NB: in asyncio 3.4.1+ StreamWriter.drain() is coroutine # see bug #170 if (asyncio.iscoroutine(ret) or isinstance(ret, asyncio.Future)): yield from ret except Exception as exc: new_exc = aiohttp.ClientRequestError( 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc reader.set_exception(new_exc) self._writer = None def send(self, writer, reader): writer.set_tcp_cork(True) path = self.url.raw_path if self.url.raw_query_string: path += '?' + self.url.raw_query_string request = aiohttp.Request(writer, self.method, path, self.version) if self.compress: request.add_compression_filter(self.compress) if self.chunked is not None: request.enable_chunked_encoding() request.add_chunking_filter(self.chunked) # set default content-type if (self.method in self.POST_METHODS and hdrs.CONTENT_TYPE not in self.skip_auto_headers and hdrs.CONTENT_TYPE not in self.headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' for k, value in self.headers.items(): request.add_header(k, value) request.send_headers() self._writer = helpers.ensure_future( self.write_bytes(request, reader), loop=self.loop) self.response = self.response_class( self.method, self.url, writer=self._writer, continue100=self._continue, timeout=self._timeout) self.response._post_init(self.loop) return self.response @asyncio.coroutine def close(self): if self._writer is not None: try: yield from self._writer finally: self._writer = None def terminate(self): if self._writer is not None: if not self.loop.is_closed(): self._writer.cancel() self._writer = None
def make_mocked_request(method, path, headers=None, *, version=HttpVersion(1, 1), closing=False, app=None, writer=sentinel, payload_writer=sentinel, protocol=sentinel, transport=sentinel, payload=sentinel, sslcontext=None, client_max_size=1024**2): """ XXX copied from aiohttp but using guillotina request object Creates mocked web.Request testing purposes. Useful in unit tests, when spinning full web server is overkill or specific conditions and errors are hard to trigger. """ task = mock.Mock() loop = mock.Mock() loop.create_future.return_value = () if version < HttpVersion(1, 1): closing = True if headers is None: headers = {} if 'Host' not in headers: headers['Host'] = 'localhost' headers = CIMultiDict(headers) raw_hdrs = tuple( (k.encode('utf-8'), v.encode('utf-8')) for k, v in headers.items()) chunked = 'chunked' in headers.get(hdrs.TRANSFER_ENCODING, '').lower() message = RawRequestMessage(method, path, version, headers, raw_hdrs, closing, False, False, chunked, URL(path)) if app is None: app = test_utils._create_app_mock() if protocol is sentinel: protocol = mock.Mock() if transport is sentinel: transport = test_utils._create_transport(sslcontext) if writer is sentinel: writer = mock.Mock() writer.transport = transport if payload_writer is sentinel: payload_writer = mock.Mock() payload_writer.write_eof.side_effect = noop payload_writer.drain.side_effect = noop protocol.transport = transport protocol.writer = writer if payload is sentinel: payload = mock.Mock() time_service = mock.Mock() time_service.time.return_value = 12345 time_service.strtime.return_value = "Tue, 15 Nov 1994 08:12:31 GMT" @contextmanager def timeout(*args, **kw): yield time_service.timeout = mock.Mock() time_service.timeout.side_effect = timeout req = Request(message, payload, protocol, payload_writer, time_service, task, client_max_size=client_max_size) match_info = UrlMappingMatchInfo({}, mock.Mock()) match_info.add_app(app) req._match_info = match_info return req
class Message: def __init__(self, headers=None, payload=None, from_details=None, to_details=None, contact_details=None, ): if headers: self.headers = headers else: self.headers = CIMultiDict() if from_details: self._from_details = from_details elif 'From' not in self.headers: raise ValueError('From header or from_details is required') if to_details: self._to_details = to_details elif 'To' not in self.headers: raise ValueError('To header or to_details is required') if contact_details: self._contact_details = contact_details self._payload = payload self._raw_payload = None if 'Via' not in self.headers: self.headers['Via'] = 'SIP/2.0/%(protocol)s ' + \ utils.format_host_and_port(self.contact_details['uri']['host'], self.contact_details['uri']['port']) + \ ';branch=%s' % utils.gen_branch(10) @property def payload(self): if self._payload: return self._payload elif self._raw_payload: self._payload = self._raw_payload.decode() return self._payload else: return '' @payload.setter def payload(self, payload): self._payload = payload @property def from_details(self): if not hasattr(self, '_from_details'): self._from_details = Contact.from_header(self.headers['From']) return self._from_details @from_details.setter def from_details(self, from_details): self._from_details = from_details @property def to_details(self): if not hasattr(self, '_to_details'): self._to_details = Contact.from_header(self.headers['To']) return self._to_details @to_details.setter def to_details(self, to_details): self._to_details = to_details @property def contact_details(self): if not hasattr(self, '_contact_details'): if 'Contact' in self.headers: self._contact_details = Contact.from_header(self.headers['Contact']) else: self._contact_details = None return self._contact_details @contact_details.setter def contact_details(self, contact_details): self._contact_details = contact_details @property def content_type(self): return self.headers['Content-Type'] @content_type.setter def content_type(self, content_type): self.headers['Content-Type'] = content_type @property def cseq(self): if not hasattr(self, '_cseq'): self._cseq = int(self.headers['CSeq'].split(' ')[0]) return self._cseq @cseq.setter def cseq(self, cseq): self._cseq = int(cseq) @property def method(self): if not hasattr(self, '_method'): self._method = self.headers['CSeq'].split(' ')[1] return self._method @method.setter def method(self, method): self._method = method def __str__(self): if self._payload: self._raw_payload = self._payload.encode() elif not self._raw_payload: self._raw_payload = b'' msg = self._make_headers() return msg + self.payload def encode(self, *args, **kwargs): if self._payload: self._raw_payload = self._payload.encode(*args, **kwargs) elif not self._raw_payload: self._raw_payload = b'' msg = self._make_headers() return msg.encode(*args, **kwargs) + self._raw_payload def _make_headers(self): if hasattr(self, '_from_details'): self.headers['From'] = str(self.from_details) if hasattr(self, '_to_details'): self.headers['To'] = str(self.to_details) if hasattr(self, '_contact_details'): self.headers['Contact'] = str(self.contact_details) if hasattr(self, '_cseq'): self.headers['CSeq'] = '%s %s' % (self.cseq, self.method) elif hasattr(self, '_method'): self.headers['CSeq'] = '%s %s' % (self.cseq, self.method) self.headers['Content-Length'] = str(len(self._raw_payload)) if 'Max-Forwards' not in self.headers: self.headers['Max-Forwards'] = '70' if 'Call-ID' not in self.headers: self.headers['Call-ID'] = uuid.uuid4() return self._format_headers() def _format_headers(self): msg = [] for k, v in sorted(self.headers.items()): if k == 'Via': if isinstance(v, (list, tuple)): msg = ['%s: %s' % (k, i) for i in v] + msg else: msg.insert(0, '%s: %s' % (k, v)) else: if isinstance(v, (list, tuple)): msg.extend(['%s: %s' % (k, i) for i in v]) else: msg.append('%s: %s' % (k, v)) msg.append(utils.EOL) return utils.EOL.join(msg) def parsed_xml(self): if 'Content-Type' not in self.headers: return None if not self.headers['Content-Type'].endswith('+xml'): return None return PyQuery(self.payload).remove_namespaces() @classmethod def from_raw_headers(cls, raw_headers): headers = CIMultiDict() decoded_headers = raw_headers.decode().split(utils.EOL) for line in decoded_headers[1:]: k, v = line.split(': ', 1) if k in headers: o = headers.setdefault(k, []) if not isinstance(o, list): o = [o] o.append(v) headers[k] = o else: headers[k] = v m = FIRST_LINE_PATTERN['response']['regex'].match(decoded_headers[0]) if m: d = m.groupdict() return Response(status_code=int(d['status_code']), status_message=d['status_message'], headers=headers, first_line=decoded_headers[0]) else: m = FIRST_LINE_PATTERN['request']['regex'].match(decoded_headers[0]) if m: d = m.groupdict() cseq, _ = headers['CSeq'].split() return Request(method=d['method'], headers=headers, cseq=int(cseq), first_line=decoded_headers[0]) else: LOG.debug(decoded_headers) raise ValueError('Not a SIP message')
class HttpMessage(PayloadWriter): """HttpMessage allows to write headers and payload to a stream.""" HOP_HEADERS = None # Must be set by subclass. SERVER_SOFTWARE = 'Python/{0[0]}.{0[1]} aiohttp/{1}'.format( sys.version_info, aiohttp.__version__) upgrade = False # Connection: UPGRADE websocket = False # Upgrade: WEBSOCKET has_chunked_hdr = False # Transfer-encoding: chunked def __init__(self, transport, version, close, loop=None): super().__init__(transport, loop) self.version = version self.closing = close self.keepalive = None self.length = None self.headers = CIMultiDict() self.headers_sent = False @property def body_length(self): return self.output_length def force_close(self): self.closing = True self.keepalive = False def keep_alive(self): if self.keepalive is None: if self.version < HttpVersion10: # keep alive not supported at all return False if self.version == HttpVersion10: if self.headers.get(hdrs.CONNECTION) == 'keep-alive': return True else: # no headers means we close for Http 1.0 return False else: return not self.closing else: return self.keepalive def is_headers_sent(self): return self.headers_sent def add_header(self, name, value): """Analyze headers. Calculate content length, removes hop headers, etc.""" assert not self.headers_sent, 'headers have been sent already' assert isinstance(name, str), \ 'Header name should be a string, got {!r}'.format(name) assert set(name).issubset(ASCIISET), \ 'Header name should contain ASCII chars, got {!r}'.format(name) assert isinstance(value, str), \ 'Header {!r} should have string value, got {!r}'.format( name, value) name = istr(name) value = value.strip() if name == hdrs.CONTENT_LENGTH: self.length = int(value) if name == hdrs.TRANSFER_ENCODING: self.has_chunked_hdr = value.lower() == 'chunked' if name == hdrs.CONNECTION: val = value.lower() # handle websocket if 'upgrade' in val: self.upgrade = True # connection keep-alive elif 'close' in val: self.keepalive = False elif 'keep-alive' in val: self.keepalive = True elif name == hdrs.UPGRADE: if 'websocket' in value.lower(): self.websocket = True self.headers[name] = value elif name not in self.HOP_HEADERS: # ignore hop-by-hop headers self.headers.add(name, value) def add_headers(self, *headers): """Adds headers to a HTTP message.""" for name, value in headers: self.add_header(name, value) def send_headers(self, _sep=': ', _end='\r\n'): """Writes headers to a stream. Constructs payload writer.""" # Chunked response is only for HTTP/1.1 clients or newer # and there is no Content-Length header is set. # Do not use chunked responses when the response is guaranteed to # not have a response body (304, 204). assert not self.headers_sent, 'headers have been sent already' self.headers_sent = True if not self.chunked and self.autochunked(): self.enable_chunking() if self.chunked: self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' self._add_default_headers() # status + headers headers = self.status_line + ''.join( [k + _sep + v + _end for k, v in self.headers.items()]) headers = headers.encode('utf-8') + b'\r\n' self.buffer_data(headers) def _add_default_headers(self): # set the connection header connection = None if self.upgrade: connection = 'Upgrade' elif not self.closing if self.keepalive is None else self.keepalive: if self.version == HttpVersion10: connection = 'keep-alive' else: if self.version == HttpVersion11: connection = 'close' if connection is not None: self.headers[hdrs.CONNECTION] = connection
class HTTPMessage: def __init__(self): self.version = "HTTP/1.1" self.headers = CIMultiDict() self.body = b"" self.rawform = None self.form = None self.json = None self.xml = None self.files = None self.text = None self.boundary = "--------BOUNDARY--------" def check_version(self): if not self.version.startswith("HTTP/"): raise HTTPError("HTTP version must start with HTTP/") if self.version not in ["HTTP/1.0", "HTTP/1.1"]: raise HTTPError("HTTP version not supported") def transfer_encodings(self): encoding = self.headers.get("Transfer-Encoding", "identity") return [enc.strip() for enc in encoding.split(",")] def is_chunked(self): return "chunked" in self.transfer_encodings() def parse_body(self): type, param = parseheader(self.headers.get("Content-Type", "")) is_json = type == "application/json" or type.endswith("+json") is_xml = type in XML_TYPES or type.endswith("+xml") is_text = type in TEXT_TYPES or type.startswith("text/") or is_json or is_xml if is_text: try: self.text = self.body.decode(param.get("charset", "UTF-8")) except UnicodeDecodeError: raise HTTPError("Failed to decode HTTP body") if type == "application/x-www-form-urlencoded": self.form = formdecode(self.text) self.rawform = formdecode(self.text, False) if is_json: try: self.json = json.loads(self.text) except json.JSONDecodeError: raise HTTPError("Failed to decode JSON body") if is_xml: try: self.xml = xml.parse(self.text) except ValueError as e: raise HTTPError("Failed to decode XML body: %s" %e) if type.startswith("multipart/form-data"): if "boundary" not in param: raise HTTPError("multipart/form-data required boundary parameter") self.boundary = param["boundary"] self.files = self.parse_files(self.body) def parse_files(self, data): split = b"--%s" %self.boundary.encode() parts = data.split(split) if parts[-1] != b"--\r\n" or parts[0] != b"": raise HTTPError("Failed to decode multipart body") files = MultiDict() for part in parts[1:-1]: if part[:2] != b"\r\n" or part[-2:] != b"\r\n": raise HTTPError("Failed to decode multipart body") part = part[2:-2] if not b"\r\n\r\n" in part: raise HTTPError("Failed to decode multipart body") head, body = part.split(b"\r\n\r\n", 1) try: lines = head.decode().split("\r\n") except UnicodeDecodeError: raise HTTPError("Failed to decode multipart body") headers = {} for header in lines: if not ": " in header: raise HTTPError("Invalid line in multipart headers") key, value = header.split(": ", 1) headers[key] = value if "Content-Disposition" not in headers: raise HTTPError("Expected Content-Disposition header in multipart data") type, param = parseheader(headers["Content-Disposition"]) if type != "form-data": raise HTTPError("Expected form-data header in multipart data") if "name" not in param: raise HTTPError("Expected name parameter in Content-Disposition header") files[param["name"]] = body return files def encode_body(self): text = self.text body = self.body if self.rawform is not None: if "Content-Type" not in self.headers: self.headers["Content-Type"] = "application/x-www-form-urlencoded" text = formencode(self.rawform, False) elif self.form is not None: if "Content-Type" not in self.headers: self.headers["Content-Type"] = "application/x-www-form-urlencoded" text = formencode(self.form) elif self.json is not None: if "Content-Type" not in self.headers: self.headers["Content-Type"] = "application/json" text = json.dumps(self.json) elif self.xml is not None: if "Content-Type" not in self.headers: self.headers["Content-Type"] = "application/xml" text = self.xml.encode() elif self.files is not None: if "Content-Type" not in self.headers: self.headers["Content-Type"] = "multipart/form-data" self.headers["Content-Type"] += "; boundary=%s" %self.boundary text = None body = b"" for name, data in self.files.items(): name = name.replace('"', '\\"') body += b"--%s\r\n" %self.boundary.encode() body += b"Content-Disposition: form-data; name=\"%s\"\r\n\r\n" %name.encode() body += data + b"\r\n" body += b"--%s--\r\n" %self.boundary.encode() if text is not None: if "Content-Type" not in self.headers: self.headers["Content-Type"] = "text/plain" body = text.encode() if body and "Content-Type" not in self.headers: self.headers["Content-Type"] = "application/octet-stream" if self.is_chunked(): if not body: return b"0\r\n\r\n" return b"%x\r\n" %len(body) + body + b"\r\n0\r\n\r\n" else: if body: self.headers["Content-Length"] = len(body) return body def encode_start_line(self): raise NotImplementedError("%s.encode_start_line" %self.__class__.__name__) def encode_headers(self): self.encode_body() lines = [self.encode_start_line()] for key, value in self.headers.items(): lines.append("%s: %s" %(key, value)) text = "\r\n".join(lines) + "\r\n\r\n" return text.encode() def encode(self): return self.encode_headers() + self.encode_body() @classmethod def parse(cls, data, head=False): parser = HTTPParser(cls, head) parser.update(data) parser.eof() if parser.buffer: raise HTTPError("Got more data than expected") return parser.message
def make_mocked_request(method, path, headers=None, *, version=HttpVersion(1, 1), closing=False, app=None, writer=sentinel, protocol=sentinel, transport=sentinel, payload=sentinel, sslcontext=None, secure_proxy_ssl_header=None): """Creates mocked web.Request testing purposes. Useful in unit tests, when spinning full web server is overkill or specific conditions and errors are hard to trigger. """ if version < HttpVersion(1, 1): closing = True if headers: headers = CIMultiDict(headers) raw_hdrs = tuple( (k.encode('utf-8'), v.encode('utf-8')) for k, v in headers.items()) else: headers = CIMultiDict() raw_hdrs = () chunked = 'chunked' in headers.get(hdrs.TRANSFER_ENCODING, '').lower() message = RawRequestMessage( method, path, version, headers, raw_hdrs, closing, False, False, chunked, URL(path)) if app is None: app = _create_app_mock() if protocol is sentinel: protocol = mock.Mock() if transport is sentinel: transport = _create_transport(sslcontext) if writer is sentinel: writer = mock.Mock() writer.transport = transport protocol.transport = transport protocol.writer = writer if payload is sentinel: payload = mock.Mock() time_service = mock.Mock() time_service.time.return_value = 12345 time_service.strtime.return_value = "Tue, 15 Nov 1994 08:12:31 GMT" @contextmanager def timeout(*args, **kw): yield time_service.timeout = mock.Mock() time_service.timeout.side_effect = timeout task = mock.Mock() loop = mock.Mock() loop.create_future.return_value = () req = Request(message, payload, protocol, time_service, task, loop=loop, secure_proxy_ssl_header=secure_proxy_ssl_header) match_info = UrlMappingMatchInfo({}, mock.Mock()) match_info.add_app(app) req._match_info = match_info return req
def make_mocked_request(method, path, headers=None, *, match_info=sentinel, version=HttpVersion(1, 1), closing=False, app=None, writer=sentinel, protocol=sentinel, transport=sentinel, payload=sentinel, sslcontext=None, client_max_size=1024**2, loop=...): """Creates mocked web.Request testing purposes. Useful in unit tests, when spinning full web server is overkill or specific conditions and errors are hard to trigger. """ task = mock.Mock() if loop is ...: loop = mock.Mock() loop.create_future.return_value = () if version < HttpVersion(1, 1): closing = True if headers: headers = CIMultiDict(headers) raw_hdrs = tuple( (k.encode('utf-8'), v.encode('utf-8')) for k, v in headers.items()) else: headers = CIMultiDict() raw_hdrs = () chunked = 'chunked' in headers.get(hdrs.TRANSFER_ENCODING, '').lower() message = RawRequestMessage(method, path, version, headers, raw_hdrs, closing, False, False, chunked, URL(path)) if app is None: app = _create_app_mock() if protocol is sentinel: protocol = mock.Mock() if transport is sentinel: transport = _create_transport(sslcontext) if writer is sentinel: writer = mock.Mock() writer.write_headers = make_mocked_coro(None) writer.write = make_mocked_coro(None) writer.write_eof = make_mocked_coro(None) writer.drain = make_mocked_coro(None) writer.transport = transport protocol.transport = transport protocol.writer = writer if payload is sentinel: payload = mock.Mock() req = Request(message, payload, protocol, writer, task, loop, client_max_size=client_max_size) match_info = UrlMappingMatchInfo( {} if match_info is sentinel else match_info, mock.Mock()) match_info.add_app(app) req._match_info = match_info return req
class HTTPResponse(object): def __init__(self, body=None, status=200, headers=None, content_type="text/plain", body_bytes=b""): self.content_type = content_type self.headers = headers if body is not None: self.body = self.encode_body(body) else: self.body = body_bytes self.status = status self.headers = CIMultiDict(headers or {}) def encode_body(self, data): try: return data.encode() except AttributeError: return str(data).encode() def parse_headers(self): headers = b"" for key, value in self.headers.items(): try: headers += b"%b: %b\r\n" % (key.encode(), value.encode("utf-8")) except AttributeError: headers += b"%b: %b\r\n" % (str(key).encode(), str(value).encode("utf-8")) return headers def has_message_body(self): """ According to the following RFC message body and length SHOULD NOT be included in responses status 1XX, 204 and 304. https://tools.ietf.org/html/rfc2616#section-4.4 https://tools.ietf.org/html/rfc2616#section-4.3 """ return self.status not in (204, 304) and not (100 <= self.status < 200) def output(self, version="1.1", keep_alive=False, keep_alive_timeout=None): """ TODO 先暂时不支持 keep alive :param version: :return: """ timeout_header = b"" if keep_alive and keep_alive_timeout is not None: timeout_header = b"Keep-Alive: %d\r\n" % keep_alive_timeout body = b"" if self.has_message_body(): body = self.body self.headers["Content-Length"] = self.headers.get( "Content-Length", len(self.body)) self.headers["Content-Type"] = self.headers.get( "Content-Type", self.content_type) if self.status in (304, 412): self.headers = remove_entity_headers(self.headers) headers = self.parse_headers() if self.status == 200: status_msg = b"OK" else: status_msg = STATUS_CODES.get(self.status, b"UNKNOWN RESPONSE") """ 消息结构如下: HTTP/1.1 200 OK # 状态行 Connection: close # headers Content-Type: text/plain # 空行 xxxxxbody part # body内容 """ return (b"HTTP/%b %d %b\r\n" b"Connection: %b\r\n" b"%b" b"%b\r\n" b"%b") % (version.encode(), self.status, status_msg, b"keep-alive" if keep_alive else b"close", timeout_header, headers, body)
class ClientRequest: GET_METHODS = {hdrs.METH_GET, hdrs.METH_HEAD, hdrs.METH_OPTIONS} POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} ALL_METHODS = GET_METHODS.union(POST_METHODS).union( {hdrs.METH_DELETE, hdrs.METH_TRACE}) DEFAULT_HEADERS = { hdrs.ACCEPT: '*/*', hdrs.ACCEPT_ENCODING: 'gzip, deflate', } SERVER_SOFTWARE = HttpMessage.SERVER_SOFTWARE body = b'' auth = None response = None response_class = None _writer = None # async task for streaming data _continue = None # waiter future for '100 Continue' response # N.B. # Adding __del__ method with self._writer closing doesn't make sense # because _writer is instance method, thus it keeps a reference to self. # Until writer has finished finalizer will not be called. def __init__(self, method, url, *, params=None, headers=None, skip_auto_headers=frozenset(), data=None, cookies=None, auth=None, encoding='utf-8', version=aiohttp.HttpVersion11, compress=None, chunked=None, expect100=False, loop=None, response_class=None, proxy=None, proxy_auth=None, timeout=5 * 60): if loop is None: loop = asyncio.get_event_loop() self.url = url self.method = method.upper() self.encoding = encoding self.chunked = chunked self.compress = compress self.loop = loop self.response_class = response_class or ClientResponse self._timeout = timeout if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) self.update_version(version) self.update_host(url) self.update_path(params) self.update_headers(headers) self.update_auto_headers(skip_auto_headers) self.update_cookies(cookies) self.update_content_encoding(data) self.update_auth(auth) self.update_proxy(proxy, proxy_auth) self.update_body_from_data(data, skip_auto_headers) self.update_transfer_encoding() self.update_expect_continue(expect100) def update_host(self, url): """Update destination host, port and connection type (ssl).""" url_parsed = urllib.parse.urlsplit(url) # check for network location part netloc = url_parsed.netloc if not netloc: raise ValueError('Host could not be detected.') # get host/port host = url_parsed.hostname if not host: raise ValueError('Host could not be detected.') try: port = url_parsed.port except ValueError: raise ValueError('Port number could not be converted.') from None # check domain idna encoding try: host = host.encode('idna').decode('utf-8') netloc = self.make_netloc(host, url_parsed.port) except UnicodeError: raise ValueError('URL has an invalid label.') # basic auth info username, password = url_parsed.username, url_parsed.password if username: self.auth = helpers.BasicAuth(username, password or '') # Record entire netloc for usage in host header self.netloc = netloc scheme = url_parsed.scheme self.ssl = scheme in ('https', 'wss') # set port number if it isn't already set if not port: if self.ssl: port = HTTPS_PORT else: port = HTTP_PORT self.host, self.port, self.scheme = host, port, scheme def make_netloc(self, host, port): ret = host if port: ret = ret + ':' + str(port) return ret def update_version(self, version): """Convert request version to two elements tuple. parser HTTP version '1.1' => (1, 1) """ if isinstance(version, str): v = [l.strip() for l in version.split('.', 1)] try: version = int(v[0]), int(v[1]) except ValueError: raise ValueError( 'Can not parse http version number: {}'.format( version)) from None self.version = version def update_path(self, params): """Build path.""" # extract path scheme, netloc, path, query, fragment = urllib.parse.urlsplit(self.url) if not path: path = '/' if isinstance(params, collections.Mapping): params = list(params.items()) if params: if not isinstance(params, str): params = urllib.parse.urlencode(params) if query: query = '%s&%s' % (query, params) else: query = params self.path = urllib.parse.urlunsplit( ('', '', helpers.requote_uri(path), query, '')) self.url = urllib.parse.urlunsplit( (scheme, netloc, self.path, '', fragment)) def update_headers(self, headers): """Update request headers.""" self.headers = CIMultiDict() if headers: if isinstance(headers, dict): headers = headers.items() elif isinstance(headers, (MultiDictProxy, MultiDict)): headers = headers.items() for key, value in headers: self.headers.add(key, value) def update_auto_headers(self, skip_auto_headers): self.skip_auto_headers = skip_auto_headers used_headers = set(self.headers) | skip_auto_headers for hdr, val in self.DEFAULT_HEADERS.items(): if hdr not in used_headers: self.headers.add(hdr, val) # add host if hdrs.HOST not in used_headers: self.headers[hdrs.HOST] = self.netloc if hdrs.USER_AGENT not in used_headers: self.headers[hdrs.USER_AGENT] = self.SERVER_SOFTWARE def update_cookies(self, cookies): """Update request cookies header.""" if not cookies: return c = http.cookies.SimpleCookie() if hdrs.COOKIE in self.headers: c.load(self.headers.get(hdrs.COOKIE, '')) del self.headers[hdrs.COOKIE] if isinstance(cookies, dict): cookies = cookies.items() for name, value in cookies: if isinstance(value, http.cookies.Morsel): c[value.key] = value.value else: c[name] = value self.headers[hdrs.COOKIE] = c.output(header='', sep=';').strip() def update_content_encoding(self, data): """Set request content encoding.""" if not data: return enc = self.headers.get(hdrs.CONTENT_ENCODING, '').lower() if enc: if self.compress is not False: self.compress = enc # enable chunked, no need to deal with length self.chunked = True elif self.compress: if not isinstance(self.compress, str): self.compress = 'deflate' self.headers[hdrs.CONTENT_ENCODING] = self.compress self.chunked = True # enable chunked, no need to deal with length def update_auth(self, auth): """Set basic auth.""" if auth is None: auth = self.auth if auth is None: return if not isinstance(auth, helpers.BasicAuth): raise TypeError('BasicAuth() tuple is required instead') self.headers[hdrs.AUTHORIZATION] = auth.encode() def update_body_from_data(self, data, skip_auto_headers): if not data: return if isinstance(data, str): data = data.encode(self.encoding) if isinstance(data, (bytes, bytearray)): self.body = data if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' if hdrs.CONTENT_LENGTH not in self.headers and not self.chunked: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) elif isinstance( data, (asyncio.StreamReader, streams.StreamReader, streams.DataQueue)): self.body = data elif asyncio.iscoroutine(data): self.body = data if (hdrs.CONTENT_LENGTH not in self.headers and self.chunked is None): self.chunked = True elif isinstance(data, io.IOBase): assert not isinstance(data, io.StringIO), \ 'attempt to send text data instead of binary' self.body = data if not self.chunked and isinstance(data, io.BytesIO): # Not chunking if content-length can be determined size = len(data.getbuffer()) self.headers[hdrs.CONTENT_LENGTH] = str(size) self.chunked = False elif not self.chunked and isinstance(data, io.BufferedReader): # Not chunking if content-length can be determined try: size = os.fstat(data.fileno()).st_size - data.tell() self.headers[hdrs.CONTENT_LENGTH] = str(size) self.chunked = False except OSError: # data.fileno() is not supported, e.g. # io.BufferedReader(io.BytesIO(b'data')) self.chunked = True else: self.chunked = True if hasattr(data, 'mode'): if data.mode == 'r': raise ValueError('file {!r} should be open in binary mode' ''.format(data)) if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers and hasattr(data, 'name')): mime = mimetypes.guess_type(data.name)[0] mime = 'application/octet-stream' if mime is None else mime self.headers[hdrs.CONTENT_TYPE] = mime elif isinstance(data, MultipartWriter): self.body = data.serialize() self.headers.update(data.headers) self.chunked = self.chunked or 8192 else: if not isinstance(data, helpers.FormData): data = helpers.FormData(data) self.body = data(self.encoding) if (hdrs.CONTENT_TYPE not in self.headers and hdrs.CONTENT_TYPE not in skip_auto_headers): self.headers[hdrs.CONTENT_TYPE] = data.content_type if data.is_multipart: self.chunked = self.chunked or 8192 else: if (hdrs.CONTENT_LENGTH not in self.headers and not self.chunked): self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_transfer_encoding(self): """Analyze transfer-encoding header.""" te = self.headers.get(hdrs.TRANSFER_ENCODING, '').lower() if self.chunked: if hdrs.CONTENT_LENGTH in self.headers: del self.headers[hdrs.CONTENT_LENGTH] if 'chunked' not in te: self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' self.chunked = self.chunked if type(self.chunked) is int else 8192 else: if 'chunked' in te: self.chunked = 8192 else: self.chunked = None if hdrs.CONTENT_LENGTH not in self.headers: self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body)) def update_expect_continue(self, expect=False): if expect: self.headers[hdrs.EXPECT] = '100-continue' elif self.headers.get(hdrs.EXPECT, '').lower() == '100-continue': expect = True if expect: self._continue = helpers.create_future(self.loop) def update_proxy(self, proxy, proxy_auth): if proxy and not proxy.startswith('http://'): raise ValueError("Only http proxies are supported") if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): raise ValueError("proxy_auth must be None or BasicAuth() tuple") self.proxy = proxy self.proxy_auth = proxy_auth @asyncio.coroutine def write_bytes(self, request, reader): """Support coroutines that yields bytes objects.""" # 100 response if self._continue is not None: yield from self._continue try: if asyncio.iscoroutine(self.body): request.transport.set_tcp_nodelay(True) exc = None value = None stream = self.body while True: try: if exc is not None: result = stream.throw(exc) else: result = stream.send(value) except StopIteration as exc: if isinstance(exc.value, bytes): yield from request.write(exc.value, drain=True) break except: self.response.close() raise if isinstance(result, asyncio.Future): exc = None value = None try: value = yield result except Exception as err: exc = err elif isinstance(result, (bytes, bytearray)): yield from request.write(result, drain=True) value = None else: raise ValueError('Bytes object is expected, got: %s.' % type(result)) elif isinstance(self.body, (asyncio.StreamReader, streams.StreamReader)): request.transport.set_tcp_nodelay(True) chunk = yield from self.body.read(streams.DEFAULT_LIMIT) while chunk: yield from request.write(chunk, drain=True) chunk = yield from self.body.read(streams.DEFAULT_LIMIT) elif isinstance(self.body, streams.DataQueue): request.transport.set_tcp_nodelay(True) while True: try: chunk = yield from self.body.read() if chunk is EOF_MARKER: break yield from request.write(chunk, drain=True) except streams.EofStream: break elif isinstance(self.body, io.IOBase): chunk = self.body.read(self.chunked) while chunk: request.write(chunk) chunk = self.body.read(self.chunked) request.transport.set_tcp_nodelay(True) else: if isinstance(self.body, (bytes, bytearray)): self.body = (self.body, ) for chunk in self.body: request.write(chunk) request.transport.set_tcp_nodelay(True) except Exception as exc: new_exc = aiohttp.ClientRequestError( 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc reader.set_exception(new_exc) else: assert request.transport.tcp_nodelay try: ret = request.write_eof() # NB: in asyncio 3.4.1+ StreamWriter.drain() is coroutine # see bug #170 if (asyncio.iscoroutine(ret) or isinstance(ret, asyncio.Future)): yield from ret except Exception as exc: new_exc = aiohttp.ClientRequestError( 'Can not write request body for %s' % self.url) new_exc.__context__ = exc new_exc.__cause__ = exc reader.set_exception(new_exc) self._writer = None def send(self, writer, reader): writer.set_tcp_cork(True) request = aiohttp.Request(writer, self.method, self.path, self.version) if self.compress: request.add_compression_filter(self.compress) if self.chunked is not None: request.enable_chunked_encoding() request.add_chunking_filter(self.chunked) # set default content-type if (self.method in self.POST_METHODS and hdrs.CONTENT_TYPE not in self.skip_auto_headers and hdrs.CONTENT_TYPE not in self.headers): self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' for k, value in self.headers.items(): request.add_header(k, value) request.send_headers() self._writer = helpers.ensure_future(self.write_bytes(request, reader), loop=self.loop) self.response = self.response_class(self.method, self.url, self.host, writer=self._writer, continue100=self._continue, timeout=self._timeout) self.response._post_init(self.loop) return self.response @asyncio.coroutine def close(self): if self._writer is not None: try: yield from self._writer finally: self._writer = None def terminate(self): if self._writer is not None: if not self.loop.is_closed(): self._writer.cancel() self._writer = None
def make_mocked_request(method, path, headers=None, *, match_info=sentinel, version=HttpVersion(1, 1), closing=False, app=None, writer=sentinel, protocol=sentinel, transport=sentinel, payload=sentinel, sslcontext=None, client_max_size=1024**2, loop=...): """Creates mocked web.Request testing purposes. Useful in unit tests, when spinning full web server is overkill or specific conditions and errors are hard to trigger. """ task = mock.Mock() if loop is ...: loop = mock.Mock() loop.create_future.return_value = () if version < HttpVersion(1, 1): closing = True if headers: headers = CIMultiDict(headers) raw_hdrs = tuple( (k.encode('utf-8'), v.encode('utf-8')) for k, v in headers.items()) else: headers = CIMultiDict() raw_hdrs = () chunked = 'chunked' in headers.get(hdrs.TRANSFER_ENCODING, '').lower() message = RawRequestMessage( method, path, version, headers, raw_hdrs, closing, False, False, chunked, URL(path)) if app is None: app = _create_app_mock() if protocol is sentinel: protocol = mock.Mock() if transport is sentinel: transport = _create_transport(sslcontext) if writer is sentinel: writer = mock.Mock() writer.write_headers = make_mocked_coro(None) writer.write = make_mocked_coro(None) writer.write_eof = make_mocked_coro(None) writer.drain = make_mocked_coro(None) writer.transport = transport protocol.transport = transport protocol.writer = writer if payload is sentinel: payload = mock.Mock() req = Request(message, payload, protocol, writer, task, loop, client_max_size=client_max_size) match_info = UrlMappingMatchInfo( {} if match_info is sentinel else match_info, mock.Mock()) match_info.add_app(app) req._match_info = match_info return req
class HttpMessage(ABC): """HttpMessage allows to write headers and payload to a stream. For example, lets say we want to read file then compress it with deflate compression and then send it with chunked transfer encoding, code may look like this: >>> response = aiohttp.Response(transport, 200) We have to use deflate compression first: >>> response.add_compression_filter('deflate') Then we want to split output stream into chunks of 1024 bytes size: >>> response.add_chunking_filter(1024) We can add headers to response with add_headers() method. add_headers() does not send data to transport, send_headers() sends request/response line and then sends headers: >>> response.add_headers( ... ('Content-Disposition', 'attachment; filename="..."')) >>> response.send_headers() Now we can use chunked writer to write stream to a network stream. First call to write() method sends response status line and headers, add_header() and add_headers() method unavailable at this stage: >>> with open('...', 'rb') as f: ... chunk = fp.read(8192) ... while chunk: ... response.write(chunk) ... chunk = fp.read(8192) >>> response.write_eof() """ writer = None # 'filter' is being used for altering write() behaviour, # add_chunking_filter adds deflate/gzip compression and # add_compression_filter splits incoming data into a chunks. filter = None HOP_HEADERS = None # Must be set by subclass. SERVER_SOFTWARE = 'Python/{0[0]}.{0[1]} aiohttp/{1}'.format( sys.version_info, aiohttp.__version__) upgrade = False # Connection: UPGRADE websocket = False # Upgrade: WEBSOCKET has_chunked_hdr = False # Transfer-encoding: chunked # subclass can enable auto sending headers with write() call, # this is useful for wsgi's start_response implementation. _send_headers = False def __init__(self, transport, version, close): self.transport = transport self._version = version self.closing = close self.keepalive = None self.chunked = False self.length = None self.headers = CIMultiDict() self.headers_sent = False self.output_length = 0 self.headers_length = 0 self._output_size = 0 self._cache = {} @property @abstractmethod def status_line(self): return b'' @abstractmethod def autochunked(self): return False @property def version(self): return self._version @property def body_length(self): return self.output_length - self.headers_length def force_close(self): self.closing = True self.keepalive = False def enable_chunked_encoding(self): self.chunked = True def keep_alive(self): if self.keepalive is None: if self.version < HttpVersion10: # keep alive not supported at all return False if self.version == HttpVersion10: if self.headers.get(hdrs.CONNECTION) == 'keep-alive': return True else: # no headers means we close for Http 1.0 return False else: return not self.closing else: return self.keepalive def is_headers_sent(self): return self.headers_sent def add_header(self, name, value): """Analyze headers. Calculate content length, removes hop headers, etc.""" assert not self.headers_sent, 'headers have been sent already' assert isinstance(name, str), \ 'Header name should be a string, got {!r}'.format(name) assert set(name).issubset(ASCIISET), \ 'Header name should contain ASCII chars, got {!r}'.format(name) assert isinstance(value, str), \ 'Header {!r} should have string value, got {!r}'.format( name, value) name = istr(name) value = value.strip() if name == hdrs.CONTENT_LENGTH: self.length = int(value) if name == hdrs.TRANSFER_ENCODING: self.has_chunked_hdr = value.lower().strip() == 'chunked' if name == hdrs.CONNECTION: val = value.lower() # handle websocket if 'upgrade' in val: self.upgrade = True # connection keep-alive elif 'close' in val: self.keepalive = False elif 'keep-alive' in val: self.keepalive = True elif name == hdrs.UPGRADE: if 'websocket' in value.lower(): self.websocket = True self.headers[name] = value elif name not in self.HOP_HEADERS: # ignore hop-by-hop headers self.headers.add(name, value) def add_headers(self, *headers): """Adds headers to a HTTP message.""" for name, value in headers: self.add_header(name, value) def send_headers(self, _sep=': ', _end='\r\n'): """Writes headers to a stream. Constructs payload writer.""" # Chunked response is only for HTTP/1.1 clients or newer # and there is no Content-Length header is set. # Do not use chunked responses when the response is guaranteed to # not have a response body (304, 204). assert not self.headers_sent, 'headers have been sent already' self.headers_sent = True if self.chunked or self.autochunked(): self.writer = self._write_chunked_payload() self.headers[hdrs.TRANSFER_ENCODING] = 'chunked' elif self.length is not None: self.writer = self._write_length_payload(self.length) else: self.writer = self._write_eof_payload() next(self.writer) self._add_default_headers() # status + headers headers = self.status_line + ''.join( [k + _sep + v + _end for k, v in self.headers.items()]) headers = headers.encode('utf-8') + b'\r\n' self.output_length += len(headers) self.headers_length = len(headers) self.transport.write(headers) def _add_default_headers(self): # set the connection header connection = None if self.upgrade: connection = 'Upgrade' elif not self.closing if self.keepalive is None else self.keepalive: if self.version == HttpVersion10: connection = 'keep-alive' else: if self.version == HttpVersion11: connection = 'close' if connection is not None: self.headers[hdrs.CONNECTION] = connection def write(self, chunk, *, drain=False, EOF_MARKER=EOF_MARKER, EOL_MARKER=EOL_MARKER): """Writes chunk of data to a stream by using different writers. writer uses filter to modify chunk of data. write_eof() indicates end of stream. writer can't be used after write_eof() method being called. write() return drain future. """ assert (isinstance(chunk, (bytes, bytearray)) or chunk is EOF_MARKER), chunk size = self.output_length if self._send_headers and not self.headers_sent: self.send_headers() if self.filter: chunk = self.filter.send(chunk) while chunk not in (EOF_MARKER, EOL_MARKER): if chunk: self.writer.send(chunk) chunk = next(self.filter) else: if chunk is not EOF_MARKER: self.writer.send(chunk) self._output_size += self.output_length - size if self._output_size > 64 * 1024: if drain: self._output_size = 0 return self.transport.drain() return () def write_eof(self): self.write(EOF_MARKER) try: self.writer.throw(aiohttp.EofStream()) except StopIteration: pass return self.transport.drain() def _write_chunked_payload(self): """Write data in chunked transfer encoding.""" while True: try: chunk = yield except aiohttp.EofStream: self.transport.write(b'0\r\n\r\n') self.output_length += 5 break chunk = bytes(chunk) chunk_len = '{:x}\r\n'.format(len(chunk)).encode('ascii') self.transport.write(chunk_len + chunk + b'\r\n') self.output_length += len(chunk_len) + len(chunk) + 2 def _write_length_payload(self, length): """Write specified number of bytes to a stream.""" while True: try: chunk = yield except aiohttp.EofStream: break if length: l = len(chunk) if length >= l: self.transport.write(chunk) self.output_length += l length = length-l else: self.transport.write(chunk[:length]) self.output_length += length length = 0 def _write_eof_payload(self): while True: try: chunk = yield except aiohttp.EofStream: break self.transport.write(chunk) self.output_length += len(chunk) @wrap_payload_filter def add_chunking_filter(self, chunk_size=16*1024, *, EOF_MARKER=EOF_MARKER, EOL_MARKER=EOL_MARKER): """Split incoming stream into chunks.""" buf = bytearray() chunk = yield while True: if chunk is EOF_MARKER: if buf: yield buf yield EOF_MARKER else: buf.extend(chunk) while len(buf) >= chunk_size: chunk = bytes(buf[:chunk_size]) del buf[:chunk_size] yield chunk chunk = yield EOL_MARKER @wrap_payload_filter def add_compression_filter(self, encoding='deflate', *, EOF_MARKER=EOF_MARKER, EOL_MARKER=EOL_MARKER): """Compress incoming stream with deflate or gzip encoding.""" zlib_mode = (16 + zlib.MAX_WBITS if encoding == 'gzip' else -zlib.MAX_WBITS) zcomp = zlib.compressobj(wbits=zlib_mode) chunk = yield while True: if chunk is EOF_MARKER: yield zcomp.flush() chunk = yield EOF_MARKER else: yield zcomp.compress(chunk) chunk = yield EOL_MARKER
class Message: def __init__(self, # from_uri, # to_uri, content_type=None, headers=None, payload=None): # self.from_uri = from_uri # self.to_uri = to_uri if headers: self.headers = headers else: self.headers = CIMultiDict() for direction in ('From', 'To', 'Contact'): # parse From, To, and Contact headers direction_attribute = '%s_details' % direction.lower() if direction in self.headers: if not hasattr(self, direction_attribute): setattr(self, direction_attribute, Contact.from_header(self.headers[direction])) elif hasattr(self, direction_attribute): contact = getattr(self, direction_attribute) self.headers[direction] = str(contact) elif direction != 'Contact': raise(ValueError('You must have a "%s" header or details.' % direction)) if content_type: self.headers['Content-Type'] = content_type self.payload = payload # Build the message if 'Via' not in self.headers: self.headers['Via'] = 'SIP/2.0/%(protocol)s '+'%s:%s;branch=%s' % (self.contact_details['uri']['host'], self.contact_details['uri']['port'], utils.gen_branch(10)) if 'Max-Forwards' not in self.headers: self.headers['Max-Forwards'] = '70' if 'Call-ID' not in self.headers: self.headers['Call-ID'] = uuid.uuid4() if 'User-Agent' not in self.headers: self.headers['User-Agent'] = 'Python/{0[0]}.{0[1]}.{0[2]} aiosip/{1}'.format( sys.version_info, aiosip.__version__) if 'Content-Length' not in self.headers: payload_len = len(self.payload.encode()) if self.payload else 0 self.headers['Content-Length'] = payload_len @property def cseq(self): if not hasattr(self, '_cseq'): self._cseq = int(self.headers['CSeq'].split(' ')[0]) return self._cseq @property def method(self): if not hasattr(self, '_method'): self._method = self.headers['CSeq'].split(' ')[1] return self._method def __str__(self): msg = [] for k, v in sorted(self.headers.items()): if isinstance(v, (list, tuple)): msg.extend(['%s: %s' % (k, i) for i in v]) else: msg.append('%s: %s' % (k, v)) if self.payload: msg.append('%s%s' % (utils.EOL, self.payload)) else: msg.append(utils.EOL) return utils.EOL.join(msg) def parsed_xml(self): if 'Content-Type' not in self.headers: return None if not self.headers['Content-Type'].endswith('+xml'): return None return PyQuery(self.payload).remove_namespaces() @classmethod def from_raw_message(cls, raw_message): lines = raw_message.split(utils.EOL) first_line = lines.pop(0) headers = CIMultiDict() payload = '' reading_headers = True for line in lines: if reading_headers: if ': ' in line: k, v = line.split(': ', 1) if k in headers: o = headers.setdefault(k, []) if not isinstance(o, list): o = [o] o.append(v) headers[k] = o else: headers[k] = v else: # Finish to parse headers reading_headers = False else: # @todo: use content length to read payload payload += line # reading payload if payload == '': payload = None m = FIRST_LINE_PATTERN['response']['regex'].match(first_line) if m: d = m.groupdict() return Response(status_code=int(d['status_code']), status_message=d['status_message'], headers=headers, payload=payload) else: m = FIRST_LINE_PATTERN['request']['regex'].match(first_line) if m: d = m.groupdict() return Request(method=d['method'], headers=headers, payload=payload) else: raise ValueError('Not a SIP message')