def _getaddrinfo(self, host: str, family: int=socket.AF_UNSPEC) \ -> List[tuple]: '''Query DNS using system resolver. Coroutine. ''' event_loop = asyncio.get_event_loop() query = event_loop.getaddrinfo(host, 0, family=family, proto=socket.IPPROTO_TCP) if self._timeout: query = asyncio.wait_for(query, self._timeout) try: results = yield from query except socket.error as error: if error.errno in (socket.EAI_FAIL, socket.EAI_NODATA, socket.EAI_NONAME): raise DNSNotFound('DNS resolution failed: {error}'.format( error=error)) from error else: raise NetworkError('DNS resolution error: {error}'.format( error=error)) from error except asyncio.TimeoutError as error: raise NetworkError('DNS resolve timed out.') from error else: return results
def _connect(self): '''Connect the socket if not already connected.''' if self.connected: # Reset the callback so the context does not leak to another self._io_stream.set_close_callback(self._stream_closed_callback) return yield self._make_socket() _logger.debug('Connecting to {0}.'.format(self._resolved_address)) try: yield self._io_stream.connect(self._resolved_address, timeout=self._params.connect_timeout) except (tornado.netutil.SSLCertificateError, SSLVerficationError) as error: raise SSLVerficationError( 'Certificate error: {error}'.format(error=error)) from error except (ssl.SSLError, socket.error) as error: if error.errno == errno.ECONNREFUSED: raise ConnectionRefused('Connection refused: {error}'.format( error=error)) from error else: raise NetworkError( 'Connection error: {error}'.format(error=error)) from error else: _logger.debug('Connected.')
def _process_request(self, request, response_factory): '''Fulfill a single request. Returns: Response ''' yield self._connect() request.address = self._address self._events.pre_request(request) if sys.version_info < (3, 3): error_class = (socket.error, StreamClosedError) else: error_class = (ConnectionError, StreamClosedError) try: yield self._send_request_header(request) yield self._send_request_body(request) self._events.request.fire(request) response = yield self._read_response_header(response_factory) # TODO: handle 100 Continue yield self._read_response_body(request, response) except error_class as error: raise NetworkError('Network error: {0}'.format(error)) from error self._events.response.fire(response) raise tornado.gen.Return(response)
def _query_dns(self, host: str, family: int=socket.AF_INET) \ -> dns.resolver.Answer: '''Query DNS using Python. Coroutine. ''' record_type = {socket.AF_INET: 'A', socket.AF_INET6: 'AAAA'}[family] event_loop = asyncio.get_event_loop() query = functools.partial(self._dns_resolver.query, host, record_type, source=self._bind_address) try: answer = yield from event_loop.run_in_executor(None, query) except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as error: # dnspython doesn't raise an instance with a message, so use the # class name instead. raise DNSNotFound('DNS resolution failed: {error}'.format( error=wpull.util.get_exception_message(error))) from error except dns.exception.DNSException as error: raise NetworkError('DNS resolution error: {error}'.format( error=wpull.util.get_exception_message(error))) from error else: return answer
def _establish_tunnel(self, connection, address): '''Establish a TCP tunnel. Coroutine. ''' host = '[{}]'.format(address[0]) if ':' in address[0] else address[0] port = address[1] request = RawRequest('CONNECT', '{0}:{1}'.format(host, port)) self.add_auth_header(request) stream = Stream(connection, keep_alive=True) _logger.debug('Sending Connect.') yield From(stream.write_request(request)) _logger.debug('Read proxy response.') response = yield From(stream.read_response()) if response.status_code != 200: debug_file = io.BytesIO() _logger.debug('Read proxy response body.') yield From(stream.read_body(request, response, file=debug_file)) debug_file.seek(0) _logger.debug(ascii(debug_file.read())) if response.status_code == 200: connection.tunneled = True else: raise NetworkError( 'Proxy does not support CONNECT: {} {}' .format(response.status_code, wpull.string.printable_str(response.reason)) )
def read_chunk_header(self): '''Read a single chunk's header. Returns: tuple: 2-item tuple with the size of the content in the chunk and the raw header byte string. Coroutine. ''' # _logger.debug('Reading chunk.') try: chunk_size_hex = yield from self._connection.readline() except ValueError as error: raise ProtocolError( 'Invalid chunk size: {0}'.format(error)) from error if not chunk_size_hex.endswith(b'\n'): raise NetworkError('Connection closed.') try: chunk_size = int(chunk_size_hex.split(b';', 1)[0].strip(), 16) except ValueError as error: raise ProtocolError( 'Invalid chunk size: {0}'.format(error)) from error if chunk_size < 0: raise ProtocolError('Chunk size cannot be negative.') self._chunk_size = self._bytes_left = chunk_size return chunk_size, chunk_size_hex
def get(self, deadline=None): try: result = yield toro.Queue.get(self, deadline or self._deadline) except toro.Timeout as error: raise NetworkError('Read timed out.') from error raise tornado.gen.Return(result)
def _read_body_by_length(self, response, file): '''Read the connection specified by a length. Coroutine. ''' _logger.debug('Reading body by length.') file_is_async = hasattr(file, 'drain') try: body_size = int(response.fields['Content-Length']) if body_size < 0: raise ValueError('Content length cannot be negative.') except ValueError as error: _logger.warning(__( _('Invalid content length: {error}'), error=error )) yield From(self._read_body_until_close(response, file)) return bytes_left = body_size while bytes_left > 0: data = yield From(self._connection.read(self._read_size)) if not data: break bytes_left -= len(data) if bytes_left < 0: data = data[:bytes_left] _logger.warning(_('Content overrun.')) self.close() self._data_observer.notify('response_body', data) content_data = self._decompress_data(data) if file: file.write(content_data) if file_is_async: yield From(file.drain()) if bytes_left > 0: raise NetworkError('Connection closed.') content_data = self._flush_decompressor() if file and content_data: file.write(content_data) if file_is_async: yield From(file.drain())
def _update_handler(self, events): '''Update the IOLoop events to listen for.''' try: self._ioloop.update_handler(self._socket.fileno(), events) except (OSError, IOError) as error: self.close() raise NetworkError('Failed to update handler: {error}'.format( error=error)) from error
def _raise_socket_error(self): '''Get the error from the socket and raise an error.''' error_code = self._socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) self.close() if error_code == errno.ECONNREFUSED: raise ConnectionRefused(error_code, os.strerror(error_code)) else: raise NetworkError(error_code, os.strerror(error_code))
def _query(self, host, query_type): try: answer_bundle = self._resolver.query(host, query_type, raise_on_no_answer=False) return answer_bundle.rrset or () except dns.resolver.NXDOMAIN as error: # dnspython doesn't raise an instance with a message, so use the # class name instead. raise DNSNotFound('DNS resolution failed: {error}'.format( error=wpull.util.get_exception_message(error))) from error except dns.exception.DNSException as error: raise NetworkError('DNS resolution error: {error}'.format( error=wpull.util.get_exception_message(error))) from error
def _resolve_from_network(self, host, port): '''Resolve the address using network. Returns: list: A list of tuples. ''' _logger.debug('Resolving {0} {1} {2}.'.format(host, port, self._family)) try: future = self._getaddrinfo_implementation(host, port) results = yield From(trollius.wait_for(future, self._timeout)) except trollius.TimeoutError as error: raise NetworkError('DNS resolve timed out.') from error else: raise Return(results)
def resolve(self, host, port): '''Resolve the given hostname and port. Args: host (str): The hostname. port (int): The port number. Returns: tuple: A tuple containing the address that can be passed to :func:`socket.connect`. Typically, the first item is the IP address and the second item is the port number. Note that IPv6 returns a tuple containing more items than 2. ''' _logger.debug('Lookup address {0} {1}.'.format(host, port)) addresses = [] for family in self._families: results = self._get_cache(host, port, family) if results is not None: _logger.debug('DNS cache hit.') addresses.extend(results) continue future = self._resolve_tornado(host, port, family) try: results = yield wpull.util.wait_future(future, self._timeout) except wpull.util.TimedOut as error: raise NetworkError('DNS resolve timed out') from error addresses.extend(results) self._put_cache(host, port, family, results) if not addresses: raise DNSNotFound('DNS resolution did not return any results.') _logger.debug('Resolved addresses: {0}.'.format(addresses)) if self._rotate: address = random.choice(addresses) else: address = addresses[0] _logger.debug('Selected {0} as address.'.format(address)) raise tornado.gen.Return(address)
def connect_gen(self, address, server_hostname): '''Connect with timeout. Raises: :class:`.errors.NetworkError` ''' @tornado.gen.coroutine def connect(): yield tornado.gen.Task(self.super.connect, self, address, server_hostname=server_hostname) try: yield wpull.util.wait_future(connect(), self._connect_timeout) except wpull.util.TimedOut as error: self.close() raise NetworkError('Connection timed out') from error
def read_gen(self, func_name, *args, **kwargs): '''Read with timeout. Raises: :class:`.errors.NetworkError` ''' @tornado.gen.coroutine def read(): result = yield tornado.gen.Task(getattr(self.super, func_name), self, *args, **kwargs) raise tornado.gen.Return(result) try: result = yield wpull.util.wait_future(read(), self._read_timeout) except wpull.util.TimedOut as error: self.close() raise NetworkError('Read timed out.') from error else: raise tornado.gen.Return(result)
def _process_request(self, request, response_factory): '''Fulfill a single request. Returns: Response ''' yield self._connect() request.address = self._resolved_address self._events.pre_request(request) if sys.version_info < (3, 3): error_class = (socket.error, StreamClosedError, ssl.SSLError) else: error_class = (ConnectionError, StreamClosedError, ssl.SSLError) if not self._params.keep_alive and 'Connection' not in request.fields: request.fields['Connection'] = 'close' try: yield self._send_request_header(request) yield self._send_request_body(request) self._events.request.fire(request) response = yield self._read_response_header(response_factory) # TODO: handle 100 Continue yield self._read_response_body(request, response) except error_class as error: raise NetworkError('Network error: {0}'.format(error)) from error except BufferFullError as error: raise ProtocolError(*error.args) from error self._events.response.fire(response) if self.should_close(request.version, response.fields.get('Connection')): _logger.debug('HTTP connection close.') self.close() else: self._io_stream.monitor_for_close() raise tornado.gen.Return(response)
def read_response(self, response=None): '''Read the response's HTTP status line and header fields. Coroutine. ''' _logger.debug('Reading header.') if response is None: response = Response() header_lines = [] bytes_read = 0 while True: try: data = yield From(self._connection.readline()) except ValueError as error: raise ProtocolError( 'Invalid header: {0}'.format(error)) from error self._data_observer.notify('response', data) if not data.endswith(b'\n'): raise NetworkError('Connection closed.') elif data in (b'\r\n', b'\n'): break header_lines.append(data) assert data.endswith(b'\n') bytes_read += len(data) if bytes_read > 32768: raise ProtocolError('Header too big.') if not header_lines: raise ProtocolError('No header received.') response.parse(b''.join(header_lines)) raise Return(response)
def _getaddrinfo_implementation(self, host, port): '''The resolver implementation. Returns: list: A list of tuples. Each tuple contains: 1. Family (``AF_INET`` or ``AF_INET6``). 2. Address (tuple): At least two values which are IP address and port. Coroutine. ''' if self._family in (None, self.PREFER_IPv4, self.PREFER_IPv6): family_flags = socket.AF_UNSPEC else: family_flags = self._family try: results = yield From(trollius.get_event_loop().getaddrinfo( host, port, family=family_flags)) except socket.error as error: if error.errno in (socket.EAI_FAIL, socket.EAI_NODATA, socket.EAI_NONAME): raise DNSNotFound('DNS resolution failed: {error}'.format( error=error)) from error else: raise NetworkError('DNS resolution error: {error}'.format( error=error)) from error except OverflowError as error: raise DNSNotFound('DNS resolution failed: {error}'.format( error=error)) from error results = list([(result[0], result[4]) for result in results]) if self._family in (self.PREFER_IPv4, self.PREFER_IPv6): results = self.sort_results(results, self._family) raise Return(results)
def read_chunk_body(self): '''Read a fragment of a single chunk. Call :meth:`read_chunk_header` first. Returns: tuple: 2-item tuple with the content data and raw data. First item is empty bytes string when chunk is fully read. Coroutine. ''' # chunk_size = self._chunk_size bytes_left = self._bytes_left # _logger.debug(__('Getting chunk size={0}, remain={1}.', # chunk_size, bytes_left)) if bytes_left > 0: size = min(bytes_left, self._read_size) data = yield from self._connection.read(size) self._bytes_left -= len(data) return (data, data) elif bytes_left < 0: raise ProtocolError('Chunked-transfer overrun.') elif bytes_left: raise NetworkError('Connection closed.') newline_data = yield from self._connection.readline() if len(newline_data) > 2: # Should be either CRLF or LF # This could our problem or the server's problem raise ProtocolError('Error reading newline after chunk.') self._chunk_size = self._bytes_left = None return (b'', newline_data)
def _do_handshake(self, timeout): '''Do the SSL handshake and return when finished.''' while True: try: self._socket.do_handshake() except ssl.SSLError as error: if error.errno == ssl.SSL_ERROR_WANT_READ: events = yield self._wait_event(READ | ERROR, timeout=timeout) elif error.errno == ssl.SSL_ERROR_WANT_WRITE: events = yield self._wait_event(WRITE | ERROR, timeout=timeout) else: raise if events & ERROR: self._raise_socket_error() except AttributeError as error: # May occur if connection reset. Issue #98. raise NetworkError('SSL socket not ready.') from error else: break
def resolve(self, host, port): '''Resolve the given hostname and port. Args: host (str): The hostname. port (int): The port number. Returns: tuple: A tuple of length 2 where the first item is the family and the second item is address that can be passed to :func:`socket.connect`. Typically in an address, the first item is the IP family and the second item is the IP address. Note that IPv6 returns a tuple containing more items than 2. ''' _logger.debug('Lookup address {0} {1}.'.format(host, port)) addresses = [] for family in self._families: results = self._get_cache(host, port, family) if results is not None: _logger.debug('DNS cache hit.') addresses.extend(results) continue future = self._resolve_tornado(host, port, family) try: results = yield wpull. async .wait_future( future, self._timeout) except wpull. async .TimedOut as error: raise NetworkError('DNS resolve timed out') from error addresses.extend(results) self._put_cache(host, port, family, results)
def read_reply(self): '''Read a reply from the stream. Returns: .ftp.request.Reply: The reply Coroutine. ''' _logger.debug('Read reply') reply = Reply() while True: line = yield From(self._connection.readline()) if line[-1:] != b'\n': raise NetworkError('Connection closed.') self._data_observer.notify('reply', line) reply.parse(line) if reply.code is not None: break raise Return(reply)
def read_reply(self) -> Reply: '''Read a reply from the stream. Returns: .ftp.request.Reply: The reply Coroutine. ''' _logger.debug('Read reply') reply = Reply() while True: line = yield from self._connection.readline() if line[-1:] != b'\n': raise NetworkError('Connection closed.') self._data_event_dispatcher.notify_read(line) reply.parse(line) if reply.code is not None: break return reply
def run_network_operation(self, task, wait_timeout=None, close_timeout=None, name='Network operation'): '''Run the task and raise appropriate exceptions. Coroutine. ''' if wait_timeout is not None and close_timeout is not None: raise Exception( 'Cannot use wait_timeout and close_timeout at the same time') try: if close_timeout is not None: with self._close_timer.with_timeout(): data = yield from task if self._close_timer.is_timeout(): raise NetworkTimedOut( '{name} timed out.'.format(name=name)) else: return data elif wait_timeout is not None: data = yield from asyncio.wait_for(task, wait_timeout) return data else: return (yield from task) except asyncio.TimeoutError as error: self.close() raise NetworkTimedOut( '{name} timed out.'.format(name=name)) from error except (tornado.netutil.SSLCertificateError, SSLVerificationError) \ as error: self.close() raise SSLVerificationError( '{name} certificate error: {error}'.format( name=name, error=error)) from error except AttributeError as error: self.close() raise NetworkError( '{name} network error: connection closed unexpectedly: {error}' .format(name=name, error=error)) from error except (socket.error, ssl.SSLError, OSError, IOError) as error: self.close() if isinstance(error, NetworkError): raise if error.errno == errno.ECONNREFUSED: raise ConnectionRefused(error.errno, os.strerror(error.errno)) from error # XXX: This quality case brought to you by OpenSSL and Python. # Example: _ssl.SSLError: [Errno 1] error:14094418:SSL # routines:SSL3_READ_BYTES:tlsv1 alert unknown ca error_string = str(error).lower() if 'certificate' in error_string or 'unknown ca' in error_string: raise SSLVerificationError( '{name} certificate error: {error}'.format( name=name, error=error)) from error else: if error.errno: raise NetworkError(error.errno, os.strerror(error.errno)) from error else: raise NetworkError('{name} network error: {error}'.format( name=name, error=error)) from error