def _getaddrinfo(self, host: str, family: int=socket.AF_UNSPEC) \
-> List[tuple]:
'''Query DNS using system resolver.
Coroutine.
'''
event_loop = asyncio.get_event_loop()
query = event_loop.getaddrinfo(host,
0,
family=family,
proto=socket.IPPROTO_TCP)
if self._timeout:
query = asyncio.wait_for(query, self._timeout)
try:
results = yield from query
except socket.error as error:
if error.errno in (socket.EAI_FAIL, socket.EAI_NODATA,
socket.EAI_NONAME):
raise DNSNotFound('DNS resolution failed: {error}'.format(
error=error)) from error
else:
raise NetworkError('DNS resolution error: {error}'.format(
error=error)) from error
except asyncio.TimeoutError as error:
raise NetworkError('DNS resolve timed out.') from error
else:
return results
def _connect(self):
'''Connect the socket if not already connected.'''
if self.connected:
# Reset the callback so the context does not leak to another
self._io_stream.set_close_callback(self._stream_closed_callback)
return
yield self._make_socket()
_logger.debug('Connecting to {0}.'.format(self._resolved_address))
try:
yield self._io_stream.connect(self._resolved_address,
timeout=self._params.connect_timeout)
except (tornado.netutil.SSLCertificateError,
SSLVerficationError) as error:
raise SSLVerficationError(
'Certificate error: {error}'.format(error=error)) from error
except (ssl.SSLError, socket.error) as error:
if error.errno == errno.ECONNREFUSED:
raise ConnectionRefused('Connection refused: {error}'.format(
error=error)) from error
else:
raise NetworkError(
'Connection error: {error}'.format(error=error)) from error
else:
_logger.debug('Connected.')
def _process_request(self, request, response_factory):
'''Fulfill a single request.
Returns:
Response
'''
yield self._connect()
request.address = self._address
self._events.pre_request(request)
if sys.version_info < (3, 3):
error_class = (socket.error, StreamClosedError)
else:
error_class = (ConnectionError, StreamClosedError)
try:
yield self._send_request_header(request)
yield self._send_request_body(request)
self._events.request.fire(request)
response = yield self._read_response_header(response_factory)
# TODO: handle 100 Continue
yield self._read_response_body(request, response)
except error_class as error:
raise NetworkError('Network error: {0}'.format(error)) from error
self._events.response.fire(response)
raise tornado.gen.Return(response)
def _query_dns(self, host: str, family: int=socket.AF_INET) \
-> dns.resolver.Answer:
'''Query DNS using Python.
Coroutine.
'''
record_type = {socket.AF_INET: 'A', socket.AF_INET6: 'AAAA'}[family]
event_loop = asyncio.get_event_loop()
query = functools.partial(self._dns_resolver.query,
host,
record_type,
source=self._bind_address)
try:
answer = yield from event_loop.run_in_executor(None, query)
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as error:
# dnspython doesn't raise an instance with a message, so use the
# class name instead.
raise DNSNotFound('DNS resolution failed: {error}'.format(
error=wpull.util.get_exception_message(error))) from error
except dns.exception.DNSException as error:
raise NetworkError('DNS resolution error: {error}'.format(
error=wpull.util.get_exception_message(error))) from error
else:
return answer
def _establish_tunnel(self, connection, address):
'''Establish a TCP tunnel.
Coroutine.
'''
host = '[{}]'.format(address[0]) if ':' in address[0] else address[0]
port = address[1]
request = RawRequest('CONNECT', '{0}:{1}'.format(host, port))
self.add_auth_header(request)
stream = Stream(connection, keep_alive=True)
_logger.debug('Sending Connect.')
yield From(stream.write_request(request))
_logger.debug('Read proxy response.')
response = yield From(stream.read_response())
if response.status_code != 200:
debug_file = io.BytesIO()
_logger.debug('Read proxy response body.')
yield From(stream.read_body(request, response, file=debug_file))
debug_file.seek(0)
_logger.debug(ascii(debug_file.read()))
if response.status_code == 200:
connection.tunneled = True
else:
raise NetworkError(
'Proxy does not support CONNECT: {} {}'
.format(response.status_code,
wpull.string.printable_str(response.reason))
)
def read_chunk_header(self):
'''Read a single chunk's header.
Returns:
tuple: 2-item tuple with the size of the content in the chunk and
the raw header byte string.
Coroutine.
'''
# _logger.debug('Reading chunk.')
try:
chunk_size_hex = yield from self._connection.readline()
except ValueError as error:
raise ProtocolError(
'Invalid chunk size: {0}'.format(error)) from error
if not chunk_size_hex.endswith(b'\n'):
raise NetworkError('Connection closed.')
try:
chunk_size = int(chunk_size_hex.split(b';', 1)[0].strip(), 16)
except ValueError as error:
raise ProtocolError(
'Invalid chunk size: {0}'.format(error)) from error
if chunk_size < 0:
raise ProtocolError('Chunk size cannot be negative.')
self._chunk_size = self._bytes_left = chunk_size
return chunk_size, chunk_size_hex
def get(self, deadline=None):
try:
result = yield toro.Queue.get(self, deadline or self._deadline)
except toro.Timeout as error:
raise NetworkError('Read timed out.') from error
raise tornado.gen.Return(result)
def _read_body_by_length(self, response, file):
'''Read the connection specified by a length.
Coroutine.
'''
_logger.debug('Reading body by length.')
file_is_async = hasattr(file, 'drain')
try:
body_size = int(response.fields['Content-Length'])
if body_size < 0:
raise ValueError('Content length cannot be negative.')
except ValueError as error:
_logger.warning(__(
_('Invalid content length: {error}'), error=error
))
yield From(self._read_body_until_close(response, file))
return
bytes_left = body_size
while bytes_left > 0:
data = yield From(self._connection.read(self._read_size))
if not data:
break
bytes_left -= len(data)
if bytes_left < 0:
data = data[:bytes_left]
_logger.warning(_('Content overrun.'))
self.close()
self._data_observer.notify('response_body', data)
content_data = self._decompress_data(data)
if file:
file.write(content_data)
if file_is_async:
yield From(file.drain())
if bytes_left > 0:
raise NetworkError('Connection closed.')
content_data = self._flush_decompressor()
if file and content_data:
file.write(content_data)
if file_is_async:
yield From(file.drain())
def _update_handler(self, events):
'''Update the IOLoop events to listen for.'''
try:
self._ioloop.update_handler(self._socket.fileno(), events)
except (OSError, IOError) as error:
self.close()
raise NetworkError('Failed to update handler: {error}'.format(
error=error)) from error
def _raise_socket_error(self):
'''Get the error from the socket and raise an error.'''
error_code = self._socket.getsockopt(socket.SOL_SOCKET,
socket.SO_ERROR)
self.close()
if error_code == errno.ECONNREFUSED:
raise ConnectionRefused(error_code, os.strerror(error_code))
else:
raise NetworkError(error_code, os.strerror(error_code))
def _query(self, host, query_type):
try:
answer_bundle = self._resolver.query(host,
query_type,
raise_on_no_answer=False)
return answer_bundle.rrset or ()
except dns.resolver.NXDOMAIN as error:
# dnspython doesn't raise an instance with a message, so use the
# class name instead.
raise DNSNotFound('DNS resolution failed: {error}'.format(
error=wpull.util.get_exception_message(error))) from error
except dns.exception.DNSException as error:
raise NetworkError('DNS resolution error: {error}'.format(
error=wpull.util.get_exception_message(error))) from error
def _resolve_from_network(self, host, port):
'''Resolve the address using network.
Returns:
list: A list of tuples.
'''
_logger.debug('Resolving {0} {1} {2}.'.format(host, port,
self._family))
try:
future = self._getaddrinfo_implementation(host, port)
results = yield From(trollius.wait_for(future, self._timeout))
except trollius.TimeoutError as error:
raise NetworkError('DNS resolve timed out.') from error
else:
raise Return(results)
def resolve(self, host, port):
'''Resolve the given hostname and port.
Args:
host (str): The hostname.
port (int): The port number.
Returns:
tuple: A tuple containing the address that can be passed to
:func:`socket.connect`. Typically, the first item is the IP
address and the second item is the port number. Note that
IPv6 returns a tuple containing more items than 2.
'''
_logger.debug('Lookup address {0} {1}.'.format(host, port))
addresses = []
for family in self._families:
results = self._get_cache(host, port, family)
if results is not None:
_logger.debug('DNS cache hit.')
addresses.extend(results)
continue
future = self._resolve_tornado(host, port, family)
try:
results = yield wpull.util.wait_future(future, self._timeout)
except wpull.util.TimedOut as error:
raise NetworkError('DNS resolve timed out') from error
addresses.extend(results)
self._put_cache(host, port, family, results)
if not addresses:
raise DNSNotFound('DNS resolution did not return any results.')
_logger.debug('Resolved addresses: {0}.'.format(addresses))
if self._rotate:
address = random.choice(addresses)
else:
address = addresses[0]
_logger.debug('Selected {0} as address.'.format(address))
raise tornado.gen.Return(address)
def connect_gen(self, address, server_hostname):
'''Connect with timeout.
Raises:
:class:`.errors.NetworkError`
'''
@tornado.gen.coroutine
def connect():
yield tornado.gen.Task(self.super.connect,
self,
address,
server_hostname=server_hostname)
try:
yield wpull.util.wait_future(connect(), self._connect_timeout)
except wpull.util.TimedOut as error:
self.close()
raise NetworkError('Connection timed out') from error
def read_gen(self, func_name, *args, **kwargs):
'''Read with timeout.
Raises:
:class:`.errors.NetworkError`
'''
@tornado.gen.coroutine
def read():
result = yield tornado.gen.Task(getattr(self.super, func_name),
self, *args, **kwargs)
raise tornado.gen.Return(result)
try:
result = yield wpull.util.wait_future(read(), self._read_timeout)
except wpull.util.TimedOut as error:
self.close()
raise NetworkError('Read timed out.') from error
else:
raise tornado.gen.Return(result)
def _process_request(self, request, response_factory):
'''Fulfill a single request.
Returns:
Response
'''
yield self._connect()
request.address = self._resolved_address
self._events.pre_request(request)
if sys.version_info < (3, 3):
error_class = (socket.error, StreamClosedError, ssl.SSLError)
else:
error_class = (ConnectionError, StreamClosedError, ssl.SSLError)
if not self._params.keep_alive and 'Connection' not in request.fields:
request.fields['Connection'] = 'close'
try:
yield self._send_request_header(request)
yield self._send_request_body(request)
self._events.request.fire(request)
response = yield self._read_response_header(response_factory)
# TODO: handle 100 Continue
yield self._read_response_body(request, response)
except error_class as error:
raise NetworkError('Network error: {0}'.format(error)) from error
except BufferFullError as error:
raise ProtocolError(*error.args) from error
self._events.response.fire(response)
if self.should_close(request.version,
response.fields.get('Connection')):
_logger.debug('HTTP connection close.')
self.close()
else:
self._io_stream.monitor_for_close()
raise tornado.gen.Return(response)
def read_response(self, response=None):
'''Read the response's HTTP status line and header fields.
Coroutine.
'''
_logger.debug('Reading header.')
if response is None:
response = Response()
header_lines = []
bytes_read = 0
while True:
try:
data = yield From(self._connection.readline())
except ValueError as error:
raise ProtocolError(
'Invalid header: {0}'.format(error)) from error
self._data_observer.notify('response', data)
if not data.endswith(b'\n'):
raise NetworkError('Connection closed.')
elif data in (b'\r\n', b'\n'):
break
header_lines.append(data)
assert data.endswith(b'\n')
bytes_read += len(data)
if bytes_read > 32768:
raise ProtocolError('Header too big.')
if not header_lines:
raise ProtocolError('No header received.')
response.parse(b''.join(header_lines))
raise Return(response)
def _getaddrinfo_implementation(self, host, port):
'''The resolver implementation.
Returns:
list: A list of tuples.
Each tuple contains:
1. Family (``AF_INET`` or ``AF_INET6``).
2. Address (tuple): At least two values which are
IP address and port.
Coroutine.
'''
if self._family in (None, self.PREFER_IPv4, self.PREFER_IPv6):
family_flags = socket.AF_UNSPEC
else:
family_flags = self._family
try:
results = yield From(trollius.get_event_loop().getaddrinfo(
host, port, family=family_flags))
except socket.error as error:
if error.errno in (socket.EAI_FAIL, socket.EAI_NODATA,
socket.EAI_NONAME):
raise DNSNotFound('DNS resolution failed: {error}'.format(
error=error)) from error
else:
raise NetworkError('DNS resolution error: {error}'.format(
error=error)) from error
except OverflowError as error:
raise DNSNotFound('DNS resolution failed: {error}'.format(
error=error)) from error
results = list([(result[0], result[4]) for result in results])
if self._family in (self.PREFER_IPv4, self.PREFER_IPv6):
results = self.sort_results(results, self._family)
raise Return(results)
def read_chunk_body(self):
'''Read a fragment of a single chunk.
Call :meth:`read_chunk_header` first.
Returns:
tuple: 2-item tuple with the content data and raw data.
First item is empty bytes string when chunk is fully read.
Coroutine.
'''
# chunk_size = self._chunk_size
bytes_left = self._bytes_left
# _logger.debug(__('Getting chunk size={0}, remain={1}.',
# chunk_size, bytes_left))
if bytes_left > 0:
size = min(bytes_left, self._read_size)
data = yield from self._connection.read(size)
self._bytes_left -= len(data)
return (data, data)
elif bytes_left < 0:
raise ProtocolError('Chunked-transfer overrun.')
elif bytes_left:
raise NetworkError('Connection closed.')
newline_data = yield from self._connection.readline()
if len(newline_data) > 2:
# Should be either CRLF or LF
# This could our problem or the server's problem
raise ProtocolError('Error reading newline after chunk.')
self._chunk_size = self._bytes_left = None
return (b'', newline_data)
def _do_handshake(self, timeout):
'''Do the SSL handshake and return when finished.'''
while True:
try:
self._socket.do_handshake()
except ssl.SSLError as error:
if error.errno == ssl.SSL_ERROR_WANT_READ:
events = yield self._wait_event(READ | ERROR,
timeout=timeout)
elif error.errno == ssl.SSL_ERROR_WANT_WRITE:
events = yield self._wait_event(WRITE | ERROR,
timeout=timeout)
else:
raise
if events & ERROR:
self._raise_socket_error()
except AttributeError as error:
# May occur if connection reset. Issue #98.
raise NetworkError('SSL socket not ready.') from error
else:
break
def resolve(self, host, port):
'''Resolve the given hostname and port.
Args:
host (str): The hostname.
port (int): The port number.
Returns:
tuple: A tuple of length 2 where the first item is the family and
the second item is address that can be passed
to :func:`socket.connect`.
Typically in an address, the first item is the IP
family and the second item is the IP address. Note that
IPv6 returns a tuple containing more items than 2.
'''
_logger.debug('Lookup address {0} {1}.'.format(host, port))
addresses = []
for family in self._families:
results = self._get_cache(host, port, family)
if results is not None:
_logger.debug('DNS cache hit.')
addresses.extend(results)
continue
future = self._resolve_tornado(host, port, family)
try:
results = yield wpull. async .wait_future(
future, self._timeout)
except wpull. async .TimedOut as error:
raise NetworkError('DNS resolve timed out') from error
addresses.extend(results)
self._put_cache(host, port, family, results)
def read_reply(self):
'''Read a reply from the stream.
Returns:
.ftp.request.Reply: The reply
Coroutine.
'''
_logger.debug('Read reply')
reply = Reply()
while True:
line = yield From(self._connection.readline())
if line[-1:] != b'\n':
raise NetworkError('Connection closed.')
self._data_observer.notify('reply', line)
reply.parse(line)
if reply.code is not None:
break
raise Return(reply)
def read_reply(self) -> Reply:
'''Read a reply from the stream.
Returns:
.ftp.request.Reply: The reply
Coroutine.
'''
_logger.debug('Read reply')
reply = Reply()
while True:
line = yield from self._connection.readline()
if line[-1:] != b'\n':
raise NetworkError('Connection closed.')
self._data_event_dispatcher.notify_read(line)
reply.parse(line)
if reply.code is not None:
break
return reply
def run_network_operation(self,
task,
wait_timeout=None,
close_timeout=None,
name='Network operation'):
'''Run the task and raise appropriate exceptions.
Coroutine.
'''
if wait_timeout is not None and close_timeout is not None:
raise Exception(
'Cannot use wait_timeout and close_timeout at the same time')
try:
if close_timeout is not None:
with self._close_timer.with_timeout():
data = yield from task
if self._close_timer.is_timeout():
raise NetworkTimedOut(
'{name} timed out.'.format(name=name))
else:
return data
elif wait_timeout is not None:
data = yield from asyncio.wait_for(task, wait_timeout)
return data
else:
return (yield from task)
except asyncio.TimeoutError as error:
self.close()
raise NetworkTimedOut(
'{name} timed out.'.format(name=name)) from error
except (tornado.netutil.SSLCertificateError, SSLVerificationError) \
as error:
self.close()
raise SSLVerificationError(
'{name} certificate error: {error}'.format(
name=name, error=error)) from error
except AttributeError as error:
self.close()
raise NetworkError(
'{name} network error: connection closed unexpectedly: {error}'
.format(name=name, error=error)) from error
except (socket.error, ssl.SSLError, OSError, IOError) as error:
self.close()
if isinstance(error, NetworkError):
raise
if error.errno == errno.ECONNREFUSED:
raise ConnectionRefused(error.errno,
os.strerror(error.errno)) from error
# XXX: This quality case brought to you by OpenSSL and Python.
# Example: _ssl.SSLError: [Errno 1] error:14094418:SSL
# routines:SSL3_READ_BYTES:tlsv1 alert unknown ca
error_string = str(error).lower()
if 'certificate' in error_string or 'unknown ca' in error_string:
raise SSLVerificationError(
'{name} certificate error: {error}'.format(
name=name, error=error)) from error
else:
if error.errno:
raise NetworkError(error.errno,
os.strerror(error.errno)) from error
else:
raise NetworkError('{name} network error: {error}'.format(
name=name, error=error)) from error