Ejemplo n.º 1
0
    def test_connection_refused(self):
        # When a connection is refused, the connect callback should not
        # be run.  (The kqueue IOLoop used to behave differently from the
        # epoll IOLoop in this respect)
        server_socket, port = bind_unused_port()
        server_socket.close()
        stream = IOStream(socket.socket(), self.io_loop)
        self.connect_called = False

        def connect_callback():
            self.connect_called = True

        stream.set_close_callback(self.stop)
        # log messages vary by platform and ioloop implementation
        with ExpectLog(gen_log, ".*", required=False):
            stream.connect(("localhost", port), connect_callback)
            self.wait()
        self.assertFalse(self.connect_called)
        self.assertTrue(isinstance(stream.error, socket.error), stream.error)
        if sys.platform != 'cygwin':
            _ERRNO_CONNREFUSED = (errno.ECONNREFUSED, )
            if hasattr(errno, "WSAECONNREFUSED"):
                _ERRNO_CONNREFUSED += (errno.WSAECONNREFUSED, )
            # cygwin's errnos don't match those used on native windows python
            self.assertTrue(stream.error.args[0] in _ERRNO_CONNREFUSED)
Ejemplo n.º 2
0
    def connect(self):
        self._loop = IOLoop.current()
        try:
            if self.unix_socket and self.host in ('localhost', '127.0.0.1'):
                sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
                self.host_info = "Localhost via UNIX socket"
                address = self.unix_socket
            else:
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
                self.host_info = "socket %s:%d" % (self.host, self.port)
                address = (self.host, self.port)
            sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
            if self.no_delay:
                sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
            sock = IOStream(sock)
            sock.set_close_callback(self.stream_close_callback)

            child_gr = greenlet.getcurrent()
            main = child_gr.parent
            assert main is not None, "Execut must be running in child greenlet"

            if self.connect_timeout:
                def timeout():
                    if not self.socket:
                        sock.close((None, IOError("connect timeout"), None))
                self._loop.call_later(self.connect_timeout, timeout)

            def connected(future):
                if future._exc_info is not None:
                    child_gr.throw(future.exception())
                else:
                    self.socket = sock
                    child_gr.switch()

            future = sock.connect(address)
            self._loop.add_future(future, connected)
            main.switch()

            self._rfile = self.socket
            self._get_server_information()
            self._request_authentication()

            if self.sql_mode is not None:
                c = self.cursor()
                c.execute("SET sql_mode=%s", (self.sql_mode,))

            if self.init_command is not None:
                c = self.cursor()
                c.execute(self.init_command)
                self.commit()

            if self.autocommit_mode is not None:
                self.autocommit(self.autocommit_mode)
        except Exception as e:
            if self.socket:
                self._rfile = None
                self.socket.close()
                self.socket = None
            raise err.OperationalError(
                2003, "Can't connect to MySQL server on %s (%r)" % (self.unix_socket or ("%s:%s" % (self.host, self.port)), e))
Ejemplo n.º 3
0
class ForwardConnection(object):

    def __init__(self, remote_address, stream, address, headers):
        self.remote_address = remote_address
        self.stream = stream
        self.address = address
        self.headers = headers
        sock = socket.socket()
        self.remote_stream = IOStream(sock)
        self.remote_stream.connect(self.remote_address, self._on_remote_connected)    
        self.remote_stream.set_close_callback(self._on_close)    

    def _on_remote_write_complete(self):
        logging.info('send request to %s', self.remote_address)
        self.remote_stream.read_until_close(self._on_remote_read_close)

    def _on_remote_connected(self):
        logging.info('forward %r to %r', self.address, self.remote_address)
        self.remote_stream.write(self.headers, self._on_remote_write_complete)

    def _on_remote_read_close(self, data):
        self.stream.write(data, self.stream.close)

    def _on_close(self):
        logging.info('remote quit %s', self.remote_address)
        self.remote_stream.close()
Ejemplo n.º 4
0
 def test_gaierror(self):
     # Test that IOStream sets its exc_info on getaddrinfo error
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
     stream = IOStream(s, io_loop=self.io_loop)
     stream.set_close_callback(self.stop)
     stream.connect(('adomainthatdoesntexist.asdf', 54321))
     self.assertTrue(isinstance(stream.error, socket.gaierror), stream.error)
Ejemplo n.º 5
0
    def test_connection_refused(self):
        # When a connection is refused, the connect callback should not
        # be run.  (The kqueue IOLoop used to behave differently from the
        # epoll IOLoop in this respect)
        cleanup_func, port = refusing_port()
        self.addCleanup(cleanup_func)
        stream = IOStream(socket.socket(), self.io_loop)
        self.connect_called = False

        def connect_callback():
            self.connect_called = True
            self.stop()

        stream.set_close_callback(self.stop)
        # log messages vary by platform and ioloop implementation
        with ExpectLog(gen_log, ".*", required=False):
            stream.connect(("127.0.0.1", port), connect_callback)
            self.wait()
        self.assertFalse(self.connect_called)
        self.assertTrue(isinstance(stream.error, socket.error), stream.error)
        if sys.platform != "cygwin":
            _ERRNO_CONNREFUSED = (errno.ECONNREFUSED,)
            if hasattr(errno, "WSAECONNREFUSED"):
                _ERRNO_CONNREFUSED += (errno.WSAECONNREFUSED,)
            # cygwin's errnos don't match those used on native windows python
            self.assertTrue(stream.error.args[0] in _ERRNO_CONNREFUSED)
Ejemplo n.º 6
0
class ForwardConnection(object):
    def __init__(self, remote_address, stream, address, headers):
        self.remote_address = remote_address
        self.stream = stream
        self.address = address
        self.headers = headers
        sock = socket.socket()
        self.remote_stream = IOStream(sock)
        self.remote_stream.connect(self.remote_address,
                                   self._on_remote_connected)
        self.remote_stream.set_close_callback(self._on_close)

    def _on_remote_write_complete(self):
        logging.info('send request to %s', self.remote_address)
        self.remote_stream.read_until_close(self._on_remote_read_close)

    def _on_remote_connected(self):
        logging.info('forward %r to %r', self.address, self.remote_address)
        self.remote_stream.write(self.headers, self._on_remote_write_complete)

    def _on_remote_read_close(self, data):
        self.stream.write(data, self.stream.close)

    def _on_close(self):
        logging.info('remote quit %s', self.remote_address)
        self.remote_stream.close()
Ejemplo n.º 7
0
class Connection(object):
    def __init__(self, host, port, event_handler,
                 stop_after=None, io_loop=None):
        self.host = host
        self.port = port
        self._event_handler = weakref.proxy(event_handler)
        self.timeout = stop_after
        self._stream = None
        self._io_loop = io_loop
        self.try_left = 2

        self.in_progress = False
        self.read_queue = []
        self.read_callbacks = []

    def __del__(self):
        self.disconnect()

    def connect(self):
        if not self._stream:
            try:
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
                sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
                sock.settimeout(self.timeout)
                sock.connect((self.host, self.port))
                self._stream = IOStream(sock, io_loop=self._io_loop)
                self._stream.set_close_callback(self.on_stream_close)
                self.connected()
            except socket.error, e:
                raise ConnectionError(str(e))
            self.fire_event('on_connect')
Ejemplo n.º 8
0
 def test_gaierror(self):
     # Test that IOStream sets its exc_info on getaddrinfo error
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
     stream = IOStream(s, io_loop=self.io_loop)
     stream.set_close_callback(self.stop)
     stream.connect(('adomainthatdoesntexist.asdf', 54321))
     self.assertTrue(isinstance(stream.error, socket.gaierror),
                     stream.error)
Ejemplo n.º 9
0
    def _maybe_connect(self, to_pid, callback=None):
        """Asynchronously establish a connection to the remote pid."""

        callback = stack_context.wrap(callback or (lambda stream: None))

        def streaming_callback(data):
            # we are not guaranteed to get an acknowledgment, but log and discard bytes if we do.
            log.info('Received %d bytes from %s, discarding.' %
                     (len(data), to_pid))
            log.debug('  data: %r' % (data, ))

        def on_connect(exit_cb, stream):
            log.info('Connection to %s established' % to_pid)
            with self._connection_callbacks_lock:
                self._connections[to_pid] = stream
            self.__dispatch_on_connect_callbacks(to_pid, stream)
            self.__loop.add_callback(stream.read_until_close,
                                     exit_cb,
                                     streaming_callback=streaming_callback)

        create = False
        with self._connection_callbacks_lock:
            stream = self._connections.get(to_pid)
            callbacks = self._connection_callbacks.get(to_pid)

            if not stream:
                self._connection_callbacks[to_pid].append(callback)

                if not callbacks:
                    create = True

        if stream:
            self.__loop.add_callback(callback, stream)
            return

        if not create:
            return

        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
        if not sock:
            raise self.SocketError('Failed opening socket')

        stream = IOStream(sock, io_loop=self.__loop)
        stream.set_nodelay(True)
        stream.set_close_callback(
            partial(self.__on_exit, to_pid, b'reached end of stream'))

        connect_callback = partial(on_connect, partial(self.__on_exit, to_pid),
                                   stream)

        log.info('Establishing connection to %s' % to_pid)

        stream.connect((to_pid.ip, to_pid.port), callback=connect_callback)

        if stream.closed():
            raise self.SocketError('Failed to initiate stream connection')

        log.info('Maybe connected to %s' % to_pid)
Ejemplo n.º 10
0
class Connection(object):
	def __init__(self, host = "localhost", port = 6379, timeout = None, io_loop = None):
		self.host = host
		self.port = port
		self._io_loop = io_loop
		self._stream = None
		self.in_porcess = False
		self.timeout = timeout
		self._lock = 0
		self.info = {"db": 0, "pass":None}

	def __del__(self):
		self.disconnect()

	#Connect to Redis Server, use tornado.iostream.IOStream to progress write and read working
	def connect(self):
		if not self._stream:
			try:
				sock = socket.create_connection((self.host, self.port), timeout = self.timeout)
				sock.setsocketopt(socket.COL_TCP, socket.TCP_NODELAY, 1)
				self._stream = IOStream(sock, io_loop = self._io_loop)
				self._stream.set_close_callback(self.on_stream_close)
				self.info["db"] = 0
				self.info["pass"] = None
			except socket.error as e:
				raise ConnectionError(e.message)

	#the operation when stram closing
	def on_stram_close(self):
		if self._stream:
			self.disconnect()

	#close the connection
	def disconnect(self):
		if self._stream:
			s = self._stream
			self._stream = None
		try:		
			if s.socket:		
				s.socket.shutdown(socket.SHUT_RDWR)
			s.close()
		except:	
			pass

	
	#Write data		
	@gen.coroutine
	def write(self, data):
		try:	
			if nnot self._stream:
				self.disconnect()
				raise ConnectionError("Try to wrtie to non-exist Connection")
			if sys.version > "3":
				data = bytes(data, encoding = "utf-8")
			yield self._stream.write(data)
		except IOError as e:	
			raise ConnectionError(e.message)
Ejemplo n.º 11
0
class Connection(object):
    def __init__(self,
                 host,
                 port,
                 event_handler,
                 stop_after=None,
                 io_loop=None):
        self.host = host
        self.port = port
        self._event_handler = weakref.proxy(event_handler)
        self.timeout = stop_after
        self._stream = None
        self._io_loop = io_loop
        self.try_left = 2

        self.in_progress = False
        self.read_callbacks = []
        self.ready_callbacks = deque()

    def __del__(self):
        self.disconnect()

    def __enter__(self):
        return self

    def __exit__(self, *args, **kwargs):
        if self.ready_callbacks:
            # Pop a SINGLE callback from the queue and execute it.
            # The next one will be executed from the code
            # invoked by the callback
            callback = self.ready_callbacks.popleft()
            callback()

    def ready(self):
        return not self.read_callbacks and not self.ready_callbacks

    def wait_until_ready(self, callback=None):
        if callback:
            if not self.ready():
                self.ready_callbacks.append(callback)
            else:
                callback()
        return self

    def connect(self):
        if not self._stream:
            try:
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
                sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
                sock.settimeout(self.timeout)
                sock.connect((self.host, self.port))
                self._stream = IOStream(sock, io_loop=self._io_loop)
                self._stream.set_close_callback(self.on_stream_close)
                self.connected()
            except socket.error, e:
                raise ConnectionError(str(e))
            self.fire_event('on_connect')
Ejemplo n.º 12
0
    def _handle_accept(self, fd, events):
        connection, address = self._socket.accept()
        stream = IOStream(connection)
        host = "%s:%d" % address #host = ":".join(str(i) for i in address)
        self._streams[host] = stream

        ccb = functools.partial(self._handle_close, host) #same as: cb =  lambda : self._handle_close(host)
        stream.set_close_callback(ccb)
        stream.read_until("\r\n", functools.partial(self._handle_read, host))
Ejemplo n.º 13
0
  def _maybe_connect(self, to_pid, callback=None):
    """Asynchronously establish a connection to the remote pid."""

    callback = stack_context.wrap(callback or (lambda stream: None))

    def streaming_callback(data):
      # we are not guaranteed to get an acknowledgment, but log and discard bytes if we do.
      log.info('Received %d bytes from %s, discarding.' % (len(data), to_pid))
      log.debug('  data: %r' % (data,))

    def on_connect(exit_cb, stream):
      log.info('Connection to %s established' % to_pid)
      with self._connection_callbacks_lock:
        self._connections[to_pid] = stream
      self.__dispatch_on_connect_callbacks(to_pid, stream)
      self.__loop.add_callback(
          stream.read_until_close,
          exit_cb,
          streaming_callback=streaming_callback)

    create = False
    with self._connection_callbacks_lock:
      stream = self._connections.get(to_pid)
      callbacks = self._connection_callbacks.get(to_pid)

      if not stream:
        self._connection_callbacks[to_pid].append(callback)

        if not callbacks:
          create = True

    if stream:
      self.__loop.add_callback(callback, stream)
      return

    if not create:
      return

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
    if not sock:
      raise self.SocketError('Failed opening socket')

    stream = IOStream(sock, io_loop=self.__loop)
    stream.set_nodelay(True)
    stream.set_close_callback(partial(self.__on_exit, to_pid, b'reached end of stream'))

    connect_callback = partial(on_connect, partial(self.__on_exit, to_pid), stream)

    log.info('Establishing connection to %s' % to_pid)

    stream.connect((to_pid.ip, to_pid.port), callback=connect_callback)

    if stream.closed():
      raise self.SocketError('Failed to initiate stream connection')

    log.info('Maybe connected to %s' % to_pid)
Ejemplo n.º 14
0
class Connection(object):
    def __init__(self, host='localhost', port=6379, weak_event_handler=None,
                 stop_after=None, io_loop=None):
        self.host = host
        self.port = port
        if weak_event_handler:
            self._event_handler = weak_event_handler
        else:
            self._event_handler = None
        self.timeout = stop_after
        self._stream = None
        self._io_loop = io_loop

        self.in_progress = False
        self.read_callbacks = []
        self.ready_callbacks = deque()
        self._lock = 0
        self.info = {'db': 0}

    def __del__(self):
        self.disconnect()

    def execute_pending_command(self):
        # Continue with the pending command execution
        # if all read operations are completed.
        if not self.read_callbacks and self.ready_callbacks:
            # Pop a SINGLE callback from the queue and execute it.
            # The next one will be executed from the code
            # invoked by the callback
            callback = self.ready_callbacks.popleft()
            callback()

    def ready(self):
        return (not self.read_callbacks and
                not self.ready_callbacks)

    def wait_until_ready(self, callback=None):
        if callback:
            if not self.ready():
                self.ready_callbacks.append(callback)
            else:
                callback()

    def connect(self):
        if not self._stream:
            try:
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
                sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
                sock.settimeout(self.timeout)
                sock.connect((self.host, self.port))
                self._stream = IOStream(sock, io_loop=self._io_loop)
                self._stream.set_close_callback(self.on_stream_close)
                self.info['db'] = 0
            except socket.error, e:
                raise ConnectionError(str(e))
            self.fire_event('on_connect')
Ejemplo n.º 15
0
 def test_gaierror(self):
     # Test that IOStream sets its exc_info on getaddrinfo error
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
     stream = IOStream(s, io_loop=self.io_loop)
     stream.set_close_callback(self.stop)
     # To reliably generate a gaierror we use a malformed domain name
     # instead of a name that's simply unlikely to exist (since
     # opendns and some ISPs return bogus addresses for nonexistent
     # domains instead of the proper error codes).
     stream.connect(('an invalid domain', 54321))
     self.assertTrue(isinstance(stream.error, socket.gaierror), stream.error)
Ejemplo n.º 16
0
 def get_stream(self):
     "Return IOStream or SSLIOstream to VNC server"
     import socket
     import commands
     from tornado.iostream import IOStream
     sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
     stream = IOStream(sock)
     stream.set_close_callback(self.on_close)
     stream.connect(
         ('127.0.0.1', self.get_vnc_port()), self.connected_to_vnc)
     return stream
Ejemplo n.º 17
0
 def test_gaierror(self):
     # Test that IOStream sets its exc_info on getaddrinfo error
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
     stream = IOStream(s, io_loop=self.io_loop)
     stream.set_close_callback(self.stop)
     # To reliably generate a gaierror we use a malformed domain name
     # instead of a name that's simply unlikely to exist (since
     # opendns and some ISPs return bogus addresses for nonexistent
     # domains instead of the proper error codes).
     stream.connect(('an invalid domain', 54321))
     self.assertTrue(isinstance(stream.error, socket.gaierror), stream.error)
Ejemplo n.º 18
0
class Connection(object):
    def __init__(self, host, port, event_handler,
                 stop_after=None, io_loop=None):
        self.host = host
        self.port = port
        self._event_handler = weakref.proxy(event_handler)
        self.timeout = stop_after
        self._stream = None
        self._io_loop = io_loop
        self.try_left = 2

        self.in_progress = False
        self.read_callbacks = []
        self.ready_callbacks = deque()

    def __del__(self):
        self.disconnect()

    def __enter__(self):
        return self

    def __exit__(self, *args, **kwargs):
        if self.ready_callbacks:
            # Pop a SINGLE callback from the queue and execute it.
            # The next one will be executed from the code
            # invoked by the callback
            callback = self.ready_callbacks.popleft()
            callback()

    def ready(self):
        return not self.read_callbacks and not self.ready_callbacks

    def wait_until_ready(self, callback=None):
        if callback:
            if not self.ready():
                self.ready_callbacks.append(callback)
            else:
                callback()
        return self

    def connect(self):
        if not self._stream:
            try:
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
                sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
                sock.settimeout(self.timeout)
                sock.connect((self.host, self.port))
                self._stream = IOStream(sock, io_loop=self._io_loop)
                self._stream.set_close_callback(self.on_stream_close)
                self.connected()
            except socket.error, e:
                raise ConnectionError(str(e))
            self.fire_event('on_connect')
Ejemplo n.º 19
0
class Flash(object):

    def __init__(self, close_callback=None):
        self._iostream = None
        self._close_callback = close_callback

    def connect(self, host='127.0.0.1', port=9999):
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self._iostream = IOStream(sock)
        self._iostream.set_close_callback(self._on_connection_close)

        # коннектимся и начинаем слушать команды
        self._iostream.connect((host, port), self._read_head)

    def close(self):
        self._on_connection_close()

    def _on_connection_close(self):
        self._iostream.close()
        if self._close_callback:
            self._close_callback()

    def _read_head(self):
        self._iostream.read_bytes(BaseCommand.meta_size, self._on_read_head)

    def _on_read_head(self, data):
        ctype, length = struct.unpack(">BH", data)

        if length:
            self._iostream.read_bytes(length, partial(self.execute_command, ctype))
        else:
            self.execute_command(ctype)

    def execute_command(self, ctype, value=None):
        command = CommandsRegistry.get_by_type(ctype)

        if command is not None:
            command.execute(value)
        # else:
        #     print 'unknown command: type={:#x}'.format(ctype)

        self._read_head()

    @classmethod
    def start(cls, host, port):
        flash = cls(close_callback=IOLoop.instance().stop)
        flash.connect(host, port)

        signal.signal(signal.SIGINT, flash.close)

        IOLoop.instance().start()
        IOLoop.instance().close()
Ejemplo n.º 20
0
 def test_connection_refused(self):
     # When a connection is refused, the connect callback should not
     # be run.  (The kqueue IOLoop used to behave differently from the
     # epoll IOLoop in this respect)
     port = get_unused_port()
     stream = IOStream(socket.socket(), self.io_loop)
     self.connect_called = False
     def connect_callback():
         self.connect_called = True
     stream.set_close_callback(self.stop)
     stream.connect(("localhost", port), connect_callback)
     self.wait()
     self.assertFalse(self.connect_called)
Ejemplo n.º 21
0
 def test_connection_refused(self):
     # When a connection is refused, the connect callback should not
     # be run.  (The kqueue IOLoop used to behave differently from the
     # epoll IOLoop in this respect)
     port = get_unused_port()
     stream = IOStream(socket.socket(), self.io_loop)
     self.connect_called = False
     def connect_callback():
         self.connect_called = True
     stream.set_close_callback(self.stop)
     stream.connect(("localhost", port), connect_callback)
     self.wait()
     self.assertFalse(self.connect_called)
Ejemplo n.º 22
0
    def connect():
        logging.debug('testing rtsp netcam at %s' % url)

        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
        s.settimeout(settings.MJPG_CLIENT_TIMEOUT)
        stream = IOStream(s)
        stream.set_close_callback(on_close)
        stream.connect((data['host'], int(data['port'])), on_connect)
        
        timeout[0] = io_loop.add_timeout(datetime.timedelta(seconds=settings.MJPG_CLIENT_TIMEOUT),
                functools.partial(on_connect, _timeout=True))
        
        return stream
Ejemplo n.º 23
0
    def connect():
        logging.debug('testing rtsp netcam at %s' % url)

        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
        s.settimeout(settings.MJPG_CLIENT_TIMEOUT)
        stream = IOStream(s)
        stream.set_close_callback(on_close)
        stream.connect((data['host'], int(data['port'])), on_connect)
        
        timeout[0] = io_loop.add_timeout(datetime.timedelta(seconds=settings.MJPG_CLIENT_TIMEOUT),
                functools.partial(on_connect, _timeout=True))
        
        return stream
class _HTTPConnection(simple_httpclient._HTTPConnection):
    def __init__(self, *args, **kwargs):
        self.source_address = kwargs.pop('source_address', None)
        super(_HTTPConnection, self).__init__(*args, **kwargs)

    def _on_resolve(self, addrinfo):
        af, sockaddr = addrinfo[0]

        if self.parsed.scheme == "https":
            ssl_options = {}
            if self.request.validate_cert:
                ssl_options["cert_reqs"] = ssl.CERT_REQUIRED
            if self.request.ca_certs is not None:
                ssl_options["ca_certs"] = self.request.ca_certs
            else:
                ssl_options["ca_certs"] = simple_httpclient._DEFAULT_CA_CERTS
            if self.request.client_key is not None:
                ssl_options["keyfile"] = self.request.client_key
            if self.request.client_cert is not None:
                ssl_options["certfile"] = self.request.client_cert

            if sys.version_info >= (2, 7):
                ssl_options["ciphers"] = "DEFAULT:!SSLv2"
            else:
                # This is really only necessary for pre-1.0 versions
                # of openssl, but python 2.6 doesn't expose version
                # information.
                ssl_options["ssl_version"] = ssl.PROTOCOL_SSLv3

            self.stream = SSLIOStream(socket.socket(af),
                                      io_loop=self.io_loop,
                                      ssl_options=ssl_options,
                                      max_buffer_size=self.max_buffer_size)
        else:
            self.stream = IOStream(socket.socket(af),
                                   io_loop=self.io_loop,
                                   max_buffer_size=self.max_buffer_size)

        if self.source_address:
            self.stream.socket.bind(self.source_address)

        timeout = min(self.request.connect_timeout, self.request.request_timeout)
        if timeout:
            self._timeout = self.io_loop.add_timeout(
                self.start_time + timeout,
                stack_context.wrap(self._on_timeout))
        self.stream.set_close_callback(self._on_close)
        # ipv6 addresses are broken (in self.parsed.hostname) until
        # 2.7, here is correctly parsed value calculated in __init__
        self.stream.connect(sockaddr, self._on_connect,
                            server_hostname=self.parsed_hostname)
Ejemplo n.º 25
0
class RemoteUpstream(Upstream):

    """
    The most methods are the same in LocalUpstream, but maybe in future
    need to be diffrent.

    """

    def initialize(self):
        self.socket = socket.socket(self._address_type, socket.SOCK_STREAM)
        self.stream = IOStream(self.socket)
        self.stream.set_close_callback(self.on_close)

    def do_connect(self):
        self.stream.connect(self.dest, self.on_connect)

    @property
    def address(self):
        return self.socket.getsockname()

    @property
    def address_type(self):
        return self._address_type

    def on_connect(self):

        self.connection_callback(self)
        on_finish = functools.partial(self.on_streaming_data, finished=True)
        self.stream.read_until_close(on_finish, self.on_streaming_data)

    def on_close(self):
        if self.stream.error:
            self.error_callback(self, self.stream.error)
        else:
            self.close_callback(self)

    def on_streaming_data(self, data, finished=False):
        if len(data):
            self.streaming_callback(self, data)

    def do_write(self, data):
        try:
            self.stream.write(data)
        except IOError as e:
            self.close()

    def do_close(self):
        if self.socket:
            logger.info("close upstream: %s:%s" % self.address)
            self.stream.close()
Ejemplo n.º 26
0
 def test_gaierror(self):
     # Test that IOStream sets its exc_info on getaddrinfo error.
     # It's difficult to reliably trigger a getaddrinfo error;
     # some resolvers own't even return errors for malformed names,
     # so we mock it instead. If IOStream changes to call a Resolver
     # before sock.connect, the mock target will need to change too.
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
     stream = IOStream(s)
     stream.set_close_callback(self.stop)
     with mock.patch('socket.socket.connect',
                     side_effect=socket.gaierror(errno.EIO, 'boom')):
         with self.assertRaises(StreamClosedError):
             yield stream.connect(('localhost', 80))
         self.assertTrue(isinstance(stream.error, socket.gaierror))
Ejemplo n.º 27
0
 def test_gaierror(self):
     # Test that IOStream sets its exc_info on getaddrinfo error.
     # It's difficult to reliably trigger a getaddrinfo error;
     # some resolvers own't even return errors for malformed names,
     # so we mock it instead. If IOStream changes to call a Resolver
     # before sock.connect, the mock target will need to change too.
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
     stream = IOStream(s)
     stream.set_close_callback(self.stop)
     with mock.patch("socket.socket.connect",
                     side_effect=socket.gaierror(errno.EIO, "boom")):
         with self.assertRaises(StreamClosedError):
             yield stream.connect(("10.0.0.7", 80))
         self.assertTrue(isinstance(stream.error, socket.gaierror))
Ejemplo n.º 28
0
class BeanstalktClient(beanstalkt.Client):
    def _reconnect(self):
        pass

    @gen.coroutine
    def connect(self):
        """Connect to beanstalkd server."""
        if not self.closed():
            return
        self._talking = False
        self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
        self._stream = IOStream(self._socket)
        self._stream.set_close_callback(self._reconnect)
        yield self._stream.connect((self.host, self.port))
Ejemplo n.º 29
0
 def _connect_to_node(self, host, data=None):
     try:
         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
         sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
         address = host.split(':')
         sock.connect(tuple((address[0], int(address[1]))))
         stream = IOStream(sock, io_loop=ioloop.IOLoop.instance())
         self._streams[host] = stream
         stream.set_close_callback(functools.partial(self._handle_close, host))
         self._streams[host].read_until("\r\n", functools.partial(self._handle_read, host))
         if data:
             stream.write(data)
     except socket.error, e:
         a = 5
Ejemplo n.º 30
0
 def test_gaierror(self):
     # Test that IOStream sets its exc_info on getaddrinfo error.
     # It's difficult to reliably trigger a getaddrinfo error;
     # some resolvers own't even return errors for malformed names,
     # so we mock it instead. If IOStream changes to call a Resolver
     # before sock.connect, the mock target will need to change too.
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
     stream = IOStream(s, io_loop=self.io_loop)
     stream.set_close_callback(self.stop)
     with mock.patch('socket.socket.connect',
                     side_effect=socket.gaierror('boom')):
         with ExpectLog(gen_log, "Connect error"):
             stream.connect(('localhost', 80), callback=self.stop)
             self.wait()
             self.assertIsInstance(stream.error, socket.gaierror)
Ejemplo n.º 31
0
    def test_connection_refused(self: typing.Any):
        # When a connection is refused, the connect callback should not
        # be run.  (The kqueue IOLoop used to behave differently from the
        # epoll IOLoop in this respect)
        cleanup_func, port = refusing_port()
        self.addCleanup(cleanup_func)
        stream = IOStream(socket.socket())

        stream.set_close_callback(self.stop)
        # log messages vary by platform and ioloop implementation
        with ExpectLog(gen_log, ".*", required=False):
            with self.assertRaises(StreamClosedError):
                yield stream.connect(("127.0.0.1", port))

        self.assertTrue(isinstance(stream.error, ConnectionRefusedError), stream.error)
Ejemplo n.º 32
0
 def test_gaierror(self):
     # Test that IOStream sets its exc_info on getaddrinfo error.
     # It's difficult to reliably trigger a getaddrinfo error;
     # some resolvers own't even return errors for malformed names,
     # so we mock it instead. If IOStream changes to call a Resolver
     # before sock.connect, the mock target will need to change too.
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
     stream = IOStream(s)
     stream.set_close_callback(self.stop)
     with mock.patch('socket.socket.connect',
                     side_effect=socket.gaierror(errno.EIO, 'boom')):
         with ExpectLog(gen_log, "Connect error"):
             stream.connect(('localhost', 80), callback=self.stop)
             self.wait()
             self.assertIsInstance(stream.error, socket.gaierror)
Ejemplo n.º 33
0
class RemoteUpstream(Upstream):
    """
    The most methods are the same in LocalUpstream, but maybe in future
    need to be diffrent.

    """
    def initialize(self):
        self.socket = socket.socket(self._address_type, socket.SOCK_STREAM)
        self.stream = IOStream(self.socket)
        self.stream.set_close_callback(self.on_close)

    def do_connect(self):
        self.stream.connect(self.dest, self.on_connect)

    @property
    def address(self):
        return self.socket.getsockname()

    @property
    def address_type(self):
        return self._address_type

    def on_connect(self):

        self.connection_callback(self)
        on_finish = functools.partial(self.on_streaming_data, finished=True)
        self.stream.read_until_close(on_finish, self.on_streaming_data)

    def on_close(self):
        if self.stream.error:
            self.error_callback(self, self.stream.error)
        else:
            self.close_callback(self)

    def on_streaming_data(self, data, finished=False):
        if len(data):
            self.streaming_callback(self, data, finished)

    def do_write(self, data):
        try:
            self.stream.write(data)
        except IOError as e:
            self.close()

    def do_close(self):
        if self.socket:
            logger.debug("close upstream: %s" % self.socket)
            self.stream.close()
Ejemplo n.º 34
0
Archivo: ipc.py Proyecto: admd/salt-1
    def handle_connection(self, connection, address):
        log.trace('IPCServer: Handling connection to address: %s', address)
        try:
            kwargs = {}
            if self.opts['ipc_write_buffer'] > 0:
                kwargs['max_write_buffer_size'] = self.opts['ipc_write_buffer']
                log.trace('Setting IPC connection write buffer: %s',
                          (self.opts['ipc_write_buffer']))
            with salt.utils. async .current_ioloop(self.io_loop):
                stream = IOStream(connection, **kwargs)
            self.streams.add(stream)

            def discard_after_closed():
                self.streams.discard(stream)

            stream.set_close_callback(discard_after_closed)
Ejemplo n.º 35
0
    def connect():
        if send_auth[0]:
            logging.debug('testing rtsp netcam at %s (this time with credentials)' % url)
            
        else:
            logging.debug('testing rtsp netcam at %s' % url)

        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
        s.settimeout(settings.MJPG_CLIENT_TIMEOUT)
        stream = IOStream(s)
        stream.set_close_callback(on_close)
        stream.connect((host, int(port)), on_connect)

        timeout[0] = io_loop.add_timeout(datetime.timedelta(seconds=settings.MJPG_CLIENT_TIMEOUT),
                                         functools.partial(on_connect, _timeout=True))
        
        return stream
Ejemplo n.º 36
0
    def connect():
        if send_auth[0]:
            logging.debug('testing rtsp netcam at %s (this time with credentials)' % url)

        else:
            logging.debug('testing rtsp netcam at %s' % url)

        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
        s.settimeout(settings.MJPG_CLIENT_TIMEOUT)
        stream = IOStream(s)
        stream.set_close_callback(on_close)
        stream.connect((host, int(port)), on_connect)

        timeout[0] = io_loop.add_timeout(datetime.timedelta(seconds=settings.MJPG_CLIENT_TIMEOUT),
                                         functools.partial(on_connect, _timeout=True))

        return stream
Ejemplo n.º 37
0
    def _connect(self):
        host = self._env.config.host
        port = self._env.config.port

        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
        stream = IOStream(s)

        def callback():
            print 'connect ok, begin to recv data'
            self._recvData(stream)

            self._recvFromRelay(stream)

        def closeCallback():
            print 'connection was closed!'

        stream.set_close_callback(closeCallback)
        stream.connect((host, port), callback)
Ejemplo n.º 38
0
    def test_connection_refused(self):
        # When a connection is refused, the connect callback should not
        # be run.  (The kqueue IOLoop used to behave differently from the
        # epoll IOLoop in this respect)
        port = get_unused_port()
        stream = IOStream(socket.socket(), self.io_loop)
        self.connect_called = False

        def connect_callback():
            self.connect_called = True
        stream.set_close_callback(self.stop)
        stream.connect(("localhost", port), connect_callback)
        self.wait()
        self.assertFalse(self.connect_called)
        self.assertTrue(isinstance(stream.error, socket.error), stream.error)
        if sys.platform != 'cygwin':
            # cygwin's errnos don't match those used on native windows python
            self.assertEqual(stream.error.args[0], errno.ECONNREFUSED)
Ejemplo n.º 39
0
    def test_connection_refused(self):
        # When a connection is refused, the connect callback should not
        # be run.  (The kqueue IOLoop used to behave differently from the
        # epoll IOLoop in this respect)
        port = get_unused_port()
        stream = IOStream(socket.socket(), self.io_loop)
        self.connect_called = False

        def connect_callback():
            self.connect_called = True
        stream.set_close_callback(self.stop)
        stream.connect(("localhost", port), connect_callback)
        self.wait()
        self.assertFalse(self.connect_called)
        self.assertTrue(isinstance(stream.error, socket.error), stream.error)
        if sys.platform != 'cygwin':
            # cygwin's errnos don't match those used on native windows python
            self.assertEqual(stream.error.args[0], errno.ECONNREFUSED)
Ejemplo n.º 40
0
    def _connect(self):
        host = self._env.config.host
        port = self._env.config.port

        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
        stream = IOStream(s)

        def callback():
            print 'connect ok, begin to recv data'
            self._recvData(stream)

            self._recvFromRelay(stream)

        def closeCallback():
            print 'connection was closed!'

        stream.set_close_callback(closeCallback)
        stream.connect((host, port), callback)
Ejemplo n.º 41
0
    def handle_connection(self, connection, address):
        log.trace('IPCServer: Handling connection to address: %s', address)
        try:
            if self.opts['ipc_write_buffer'] > 0:
                log.trace('Setting IPC connection write buffer: %s',
                          (self.opts['ipc_write_buffer']))
                stream = IOStream(
                    connection,
                    io_loop=self.io_loop,
                    max_write_buffer_size=self.opts['ipc_write_buffer'])
            else:
                stream = IOStream(connection, io_loop=self.io_loop)
            self.streams.add(stream)

            def discard_after_closed():
                self.streams.discard(stream)

            stream.set_close_callback(discard_after_closed)
        except Exception as exc:
            log.error('IPC streaming error: %s', exc)
Ejemplo n.º 42
0
class Client(object):
    """ Stupid simple client """

    def __init__(self, database=0, ioloop=None):
        self._ioloop = ioloop or IOLoop.instance()
        self._database = database
        self._stream = None

    def connect(self, callback=None, host="localhost", port=6379):
        """ Connect to Redis """
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
        self._socket = sock
        self._stream = IOStream(sock, io_loop=self._ioloop)
        self._stream.set_close_callback(self._close)
        self._stream.connect((host, port), callback=callback)

    def disconnect(self):
        """ Close connection to Redis. """
        self._stream.close()

    def _close(self):
        """ Detect a close -- overwrite in sub classes if required """
        pass

    def __getattr__(self, attr):
        """ Generate a request from the attribute """
        command = attr.upper()
        if command not in VALID_COMMANDS:
            raise AttributeError("Invalid command %s" % command)
        if not self._stream:
            raise Exception("Cannot call command before connecting.")
        return Command(command, self)

    def send_message(self, message, response_class, callback):
        """ Send a message to Redis """
        self._stream.write(message, self.write_callback)
        response_class(self._stream, callback)

    def write_callback(self, *args, **kwargs):
        """ Overwrite in sub classes, if required """
        pass
Ejemplo n.º 43
0
    def test_connection_refused(self):
        # When a connection is refused, the connect callback should not
        # be run.  (The kqueue IOLoop used to behave differently from the
        # epoll IOLoop in this respect)
        cleanup_func, port = refusing_port()
        self.addCleanup(cleanup_func)
        stream = IOStream(socket.socket())

        stream.set_close_callback(self.stop)
        # log messages vary by platform and ioloop implementation
        with ExpectLog(gen_log, ".*", required=False):
            with self.assertRaises(StreamClosedError):
                yield stream.connect(("127.0.0.1", port))

        self.assertTrue(isinstance(stream.error, socket.error), stream.error)
        if sys.platform != 'cygwin':
            _ERRNO_CONNREFUSED = (errno.ECONNREFUSED, )
            if hasattr(errno, "WSAECONNREFUSED"):
                _ERRNO_CONNREFUSED += (errno.WSAECONNREFUSED, )
            # cygwin's errnos don't match those used on native windows python
            self.assertTrue(stream.error.args[0] in _ERRNO_CONNREFUSED)
Ejemplo n.º 44
0
  def maybe_connect(self, to_pid, callback=None):
    """Synchronously open a connection to to_pid or return a connection if it exists."""

    callback = stack_context.wrap(callback or (lambda stream: None))

    def streaming_callback(data):
      # we are not guaranteed to get an acknowledgment, but log and discard bytes if we do.
      log.info('Received %d bytes from %s, discarding.' % (len(data), to_pid))

    def on_connect(exit_cb, stream):
      log.info('Connection to %s established' % to_pid)
      self._connections[to_pid] = stream
      callback(stream)
      self.loop.add_callback(stream.read_until_close, exit_cb,
                             streaming_callback=streaming_callback)

    stream = self._connections.get(to_pid)

    if stream is not None:
      callback(stream)
      return

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
    if not sock:
      raise self.SocketError("Failed opening socket")

    # Set the socket non-blocking
    sock.setblocking(0)

    stream = IOStream(sock, io_loop=self.loop)
    stream.set_nodelay(True)
    stream.set_close_callback(partial(self.__on_exit, to_pid, b'closed from maybe_connect'))

    connect_callback = partial(on_connect, partial(self.__on_exit, to_pid), stream)

    log.info('Establishing connection to %s' % to_pid)
    stream.connect((to_pid.ip, to_pid.port), callback=connect_callback)
    if stream.closed():
      raise self.SocketError("Failed to initiate stream connection")
    log.info('Maybe connected to %s' % to_pid)
Ejemplo n.º 45
0
class TCPLinkClientManager(object):
    def __init__(self, config):
        self.config = config
        self.stream = None
        self.creation_callback = None

    def setup(self):
        pass

    def create(self, callback):
        self.creation_callback = callback
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
        self.stream = IOStream(s)
        self.stream.set_close_callback(self.on_close)
        self.stream.connect((self.config['host'], self.config['port']), self.on_connect)

    @tornado.gen.engine
    def on_connect(self):
        if self.stream:
            if self.creation_callback:
                link = TCPLink(self.stream)
                link.setup()
                yield tornado.gen.Task(link.establish)
                self.creation_callback(link)
                self.creation_callback = None
        self.stream = None

    def on_close(self):
        if self.stream:
            self.stream.set_close_callback(None)
            if self.stream.error:
                logging.error(self.stream.error)
            if self.creation_callback:
                self.creation_callback(None)
                self.creation_callback = None
        self.stream = None

    def cleanup(self):
        self.stream = None
Ejemplo n.º 46
0
class Client(Session):
    def __init__(self, protocol, io_loop=None):
        Session.__init__(self, protocol, io_loop)
        self.auto_reconnect = True
        self.reconnect_time = 5
        self.connect_time = 5

    def connect(self, address):
        self.status = SessionStream.CONNECTING
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
        self.ios = IOStream(sock, self.io_loop)
        self.ios.set_close_callback(self._ios_closed)
        self.address = address
        self.clear_buffer()
        self.add_timer(self._connect_timeout, self.connect_time, "connect")
        self.ios.connect(address, self._connected)

    def _connected(self):
        self.status = SessionStream.CONNECTED
        self.protocol.connected(self)
        self.remove_timer("connect")
        self.ios.read_until_close(self._disconnected, self._receiver)

    def _connect_timeout(self):
        self.remove_timer("connect")
        self.status = SessionStream.IDLE
        self.ios.close()
        if self.auto_reconnect:
            self.add_timer(self._do_reconnect, self.reconnect_time, "reconnect")

    def _ios_closed(self):
        if self.status == SessionStream.CONNECTING:
            self.remove_timer("connect")
        if self.auto_reconnect:
            self.add_timer(self._do_reconnect, self.reconnect_time, "reconnect")

    def _do_reconnect(self):
        self.remove_timer("reconnect")
        self.connect(self.address)
Ejemplo n.º 47
0
    def test_connection_refused(self):
        # When a connection is refused, the connect callback should not
        # be run.  (The kqueue IOLoop used to behave differently from the
        # epoll IOLoop in this respect)
        server_socket, port = bind_unused_port()
        server_socket.close()
        stream = IOStream(socket.socket(), self.io_loop)
        self.connect_called = False

        def connect_callback():
            self.connect_called = True

        stream.set_close_callback(self.stop)
        # log messages vary by platform and ioloop implementation
        with ExpectLog(gen_log, ".*", required=False):
            stream.connect(("localhost", port), connect_callback)
            self.wait()
        self.assertFalse(self.connect_called)
        self.assertTrue(isinstance(stream.error, socket.error), stream.error)
        if sys.platform != "cygwin":
            # cygwin's errnos don't match those used on native windows python
            self.assertEqual(stream.error.args[0], errno.ECONNREFUSED)
Ejemplo n.º 48
0
class Connection(object):
    def __init__(self, host='localhost', port=6379, weak_event_handler=None,
                 stop_after=None, io_loop=None):
        self.host = host
        self.port = port
        if weak_event_handler:
            self._event_handler = weak_event_handler
        else:
            self._event_handler = None
        self.timeout = stop_after
        self._stream = None
        self._io_loop = io_loop

        self.in_progress = False
        self.read_callbacks = []
        self.ready_callbacks = deque()
        self._lock = 0
        self.info = {}

    def __del__(self):
        self.disconnect()

    def __enter__(self):
        self._lock += 1
        return self

    def __exit__(self, *args, **kwargs):
        self._lock -= 1
        if not self._lock:
            self.continue_pending()

    def continue_pending(self):
        # Continue with the pending command execution
        # if all read operations are completed.
        if not self.read_callbacks and self.ready_callbacks:
            # Pop a SINGLE callback from the queue and execute it.
            # The next one will be executed from the code
            # invoked by the callback
            callback = self.ready_callbacks.popleft()
            callback()

    def ready(self):
        return (not self._lock and
                not self.read_callbacks and
                not self.ready_callbacks)

    def wait_until_ready(self, callback=None):
        if callback:
            if not self.ready():
                self.ready_callbacks.append(callback)
            else:
                callback()
        return self

    def connect(self):
        if not self._stream:
            try:
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
                sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
                sock.settimeout(self.timeout)
                sock.connect((self.host, self.port))
                self._stream = IOStream(sock, io_loop=self._io_loop)
                self._stream.set_close_callback(self.on_stream_close)
            except socket.error, e:
                raise ConnectionError(str(e))
            self.fire_event('on_connect')
Ejemplo n.º 49
0
    def connect(self):
        self._loop = IOLoop.current()
        try:
            if self.unix_socket and self.host in ('localhost', '127.0.0.1'):
                sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
                self.host_info = "Localhost via UNIX socket"
                address = self.unix_socket
            else:
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
                self.host_info = "socket %s:%d" % (self.host, self.port)
                address = (self.host, self.port)
            sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
            sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
            sock = IOStream(sock)
            sock.set_close_callback(self.stream_close_callback)

            child_gr = greenlet.getcurrent()
            main = child_gr.parent
            assert main is not None, "Execut must be running in child greenlet"

            if self.connect_timeout:
                def timeout():
                    if not self.socket:
                        sock.close((None, IOError("connect timeout"), None))
                self._loop.call_later(self.connect_timeout, timeout)

            def connected(future):
                if future._exc_info is not None:
                    child_gr.throw(future.exception())
                else:
                    self.socket = sock
                    child_gr.switch()

            future = sock.connect(address)
            self._loop.add_future(future, connected)
            main.switch()

            self._rfile = self.socket
            self._next_seq_id = 0

            self._get_server_information()
            self._request_authentication()

            if self.sql_mode is not None:
                c = self.cursor()
                c.execute("SET sql_mode=%s", (self.sql_mode,))

            if self.init_command is not None:
                c = self.cursor()
                c.execute(self.init_command)
                self.commit()

            if self.autocommit_mode is not None:
                self.autocommit(self.autocommit_mode)
        except Exception as e:
            if self.socket:
                self._rfile = None
                self.socket.close()
                self.socket = None
            exc = err.OperationalError(
                2003, "Can't connect to MySQL server on %s (%r)" % (self.unix_socket or ("%s:%s" % (self.host, self.port)), e))
            # Keep original exception and traceback to investigate error.
            exc.original_exception = e
            exc.traceback = traceback.format_exc()
            raise exc
Ejemplo n.º 50
0
class _RedisConnection(object):
    def __init__(self, io_loop, write_buf, final_callback, redis_tuple,
                 redis_pass):
        """
        :param io_loop: 你懂的
        :param write_buf: 第一次写入
        :param final_callback: resp赋值时调用
        :param redis_tuple: (ip, port, db)
        :param redis_pass: redis密码
        """
        self.__io_loop = io_loop
        self.__final_cb = final_callback
        self.__stream = None
        #redis应答解析remain
        self.__recv_buf = ''
        self.__write_buf = write_buf

        init_buf = ''
        init_buf = chain_select_cmd(redis_tuple[2], init_buf)
        if redis_pass is None:
            self.__init_buf = (init_buf, )
        else:
            assert redis_pass and isinstance(redis_pass, str)
            self.__init_buf = (redis_auth(redis_pass), init_buf)

        self.__haspass = redis_pass is not None
        self.__init_buf = ''.join(self.__init_buf)

        self.__connect_state = CONNECT_INIT
        #redis指令上下文, connect指令个数(AUTH, SELECT .etc),trans,cmd_count
        self.__cmd_env = deque()
        self.__written = False

    def connect(self, init_future, redis_tuple, active_trans, cmd_count):
        """
        :param init_future: 第一个future对象
        :param redis_tuple: (ip, port, db)
        :param active_trans: 事务是否激活
        :param cmd_count: 指令个数
        """
        if self.__stream is not None:
            return

        #future, connect_count, transaction, cmd_count
        self.__cmd_env.append((init_future, 1 + int(self.__haspass), False, 0))
        self.__cmd_env.append((init_future, 0, active_trans, cmd_count))

        with ExceptionStackContext(self.__handle_ex):
            self.__stream = IOStream(socket.socket(socket.AF_INET,
                                                   socket.SOCK_STREAM, 0),
                                     io_loop=self.__io_loop)
            self.__stream.set_close_callback(self.__on_close)
            self.__stream.connect(redis_tuple[:2], self.__on_connect)
            self.__connect_state = CONNECT_ING

    def connect_state(self):
        return self.__connect_state

    def write(self,
              write_buf,
              new_future,
              include_select,
              active_trans,
              cmd_count,
              by_connect=False):
        """
        :param new_future: 由于闭包的影响,在resp回调函数中会保存上一次的future对象,该对象必须得到更新
        :param include_select: 是否包含SELECT指令
        :param active_trans: 事务是否激活
        :param cmd_count: 指令个数
        """
        if by_connect:
            self.__stream.write(self.__init_buf)
            self.__init_buf = None

            if self.__write_buf:
                self.__stream.write(self.__write_buf)
                self.__write_buf = None
            return

        self.__cmd_env.append(
            (new_future, int(include_select), active_trans, cmd_count))
        if self.__connect_state == CONNECT_ING:
            self.__write_buf = ''.join((self.__write_buf, write_buf))
            return

        if self.__write_buf:
            write_buf = ''.join((self.__write_buf, write_buf))

        self.__stream.write(write_buf)
        self.__write_buf = None

    def __on_connect(self):
        """连接,只需要发送初始cmd即可
        """
        self.__connect_state = CONNECT_SUCC
        self.__stream.set_nodelay(True)
        self.write(None, None, None, None, None, True)
        self.__stream.read_until_close(None, self.__on_resp)

    def __on_resp(self, recv):
        """
        :param recv: 收到的buf
        """
        recv = ''.join((self.__recv_buf, recv))

        idx = 0
        for future, connect_count, trans, count in self.__cmd_env:
            ok, payload, recv = decode_resp_ondemand(recv, connect_count,
                                                     trans, count)
            if not ok:
                break

            idx += 1
            if count > 0:
                self.__run_callback({
                    _RESP_FUTURE: future,
                    RESP_RESULT: payload
                })

        self.__recv_buf = recv
        for _ in xrange(idx):
            self.__cmd_env.popleft()

    def __on_close(self):
        self.__connect_state = CONNECT_INIT
        if self.__final_cb:
            if self.__stream.error:
                self.__run_callback({RESP_ERR: self.__stream.error})

    def __run_callback(self, resp):
        if self.__final_cb is None:
            return

        self.__io_loop.add_callback(self.__final_cb, resp)

    def __handle_ex(self, typ, value, tb):
        """
        :param typ: 异常类型
        """
        if self.__final_cb:
            self.__run_callback({RESP_ERR: value})
            return True
        return False
Ejemplo n.º 51
0
class Connection(RedisCommandsMixin):

    def __init__(self, redis, on_connect=None):
        logger.debug('Creating new Redis connection.')
        self.redis = redis
        self.reader = hiredis.Reader()
        self._watch = set()
        self._multi = False
        self.callbacks = deque()
        self._on_connect_callback = on_connect

        self.stream = IOStream(
            socket.socket(redis._family, socket.SOCK_STREAM, 0),
            io_loop=redis._ioloop
        )
        self.stream.set_close_callback(self._on_close)
        self.stream.connect(redis._addr, self._on_connect)


    def _on_connect(self):
        logger.debug('Connected!')
        self.stream.read_until_close(self._on_close, self._on_read)
        self.redis._shared.append(self)
        if self._on_connect_callback is not None:
            self._on_connect_callback(self)
            self._on_connect_callback = None

    def _on_read(self, data):
        self.reader.feed(data)
        while True:
            resp = self.reader.gets()
            if resp is False:
                break
            callback = self.callbacks.popleft()
            if callback is not None:
                self.redis._ioloop.add_callback(partial(callback, resp))

    def is_idle(self):
        return len(self.callbacks) == 0

    def is_shared(self):
        return self in self.redis._shared

    def lock(self):
        if not self.is_shared():
            raise Exception('Connection already is locked!')
        self.redis._shared.remove(self)

    def unlock(self, callback=None):

        def cb(resp):
            assert resp == 'OK'
            self.redis._shared.append(self)

        if self._multi:
            self.send_message(['DISCARD'])
        elif self._watch:
            self.send_message(['UNWATCH'])

        self.send_message(['SELECT', self.redis._database], cb)

    def send_message(self, args, callback=None):

        command = args[0]

        if 'SUBSCRIBE' in command:
            raise NotImplementedError('Not yet.')

        # Do not allow the commands, affecting the execution of other commands,
        # to be used on shared connection.
        if command in ('WATCH', 'MULTI'):
            if self.is_shared():
                raise Exception('Command %s is not allowed while connection '
                                'is shared!' % command)
            if command == 'WATCH':
                self._watch.add(args[1])
            if command == 'MULTI':
                self._multi = True

        # monitor transaction state, to unlock correctly
        if command in ('EXEC', 'DISCARD', 'UNWATCH'):
            if command in ('EXEC', 'DISCARD'):
                self._multi = False
            self._watch.clear()

        self.stream.write(self.format_message(args))

        future = Future()

        if callback is not None:
            future.add_done_callback(stack_context.wrap(callback))

        self.callbacks.append(future.set_result)

        return future

    def format_message(self, args):
        l = "*%d" % len(args)
        lines = [l.encode('utf-8')]
        for arg in args:
            if not isinstance(arg, str):
                arg = str(arg)
            arg = arg.encode('utf-8')
            l = "$%d" % len(arg)
            lines.append(l.encode('utf-8'))
            lines.append(arg)
        lines.append(b"")
        return b"\r\n".join(lines)

    def close(self):
        self.send_command(['QUIT'])
        if self.is_shared():
            self.lock()

    def _on_close(self, data=None):
        logger.debug('Redis connection was closed.')
        if data is not None:
            self._on_read(data)
        if self.is_shared():
            self.lock()
Ejemplo n.º 52
0
class Connection(FrameHandler):
    """A "physical" TCP connection to the AMQP server

    heartbeat: int, optional
               the requested time interval in seconds for heartbeat frames.

    Connection.on_error callback, when set, is called in case of
    "hard" AMQP Error. It receives a ConnectionErrorinstance as argument:

        def handle_error(conn_error):
            print conn_error.method
            print conn_error.reply_code

        conn.on_error = handle_error

    Connection.on_disconnect callback, when set, is called in case of
    heartbeat timeout or TCP low level disconnection. It receives no args.
    """

    def __init__(self, host, username='******', password='******', vhost='/',
                 port=5672, heartbeat=0, io_loop=None):
        self.host = host
        self.port = port
        self.username = username
        self.password = password
        self.vhost = vhost
        self.heartbeat = heartbeat
        self.last_received_frame = None
        self.frame_max = 0
        self.io_loop = io_loop or IOLoop.instance()
        self.stream = None
        self.channels = {0: self}
        self.last_channel_id = 0
        self.channel_id = 0
        self.on_connect = None
        self.on_disconnect = None
        self.on_error = None
        self._close_callback = None
        self._frame_count = 0
        super(Connection, self).__init__(connection=self)

    def connect(self, callback, close_callback=None):
        """open the connection to the server"""
        if self.status is not status.CLOSED:
            raise AmqpStatusError('Connection status is %s' % self.status)
        self.status = status.OPENING
        sock = socket.socket()
        sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
        self.on_connect = callback
        self.on_disconnect = close_callback
        if TORNADO_1_2:
            self.stream = IOStream(sock, io_loop=self.io_loop)
            self.stream.set_close_callback(self.on_closed_stream)
            self.stream.connect((self.host, self.port), self._handshake)
        else:
            sock.connect((self.host, self.port))
            self.stream = IOStream(sock, io_loop=self.io_loop)
            self.stream.set_close_callback(self.on_closed_stream)
            self._handshake()

    def close(self, callback=None):
        """cleanly closes the connection to the server.

        all pending tasks are flushed before connection shutdown"""

        if self.status not in (status.OPENING, status.OPENED):
            raise AmqpStatusError('connection is not open')

        self._close_callback = callback
        self.status = status.CLOSING

        for ch in self.channels.values():
            if ch is not self and ch.status in (status.OPENING, status.OPENED):
                ch.close(self.close)

        m = Close(reply_code=0, reply_text='', class_id=0, method_id=0)
        self.send_method(m, self._close_callback)

    def _get_next_channel_id(self):
        if len(self.channels) == 0x10000:
            raise AmqpError('max channels per connection exceeded')

        next_id = self.last_channel_id
        while True:
            next_id = (next_id + 1) % 0x10000
            if next_id not in self.channels:
                break

        self.last_channel_id = next_id
        return next_id

    def channel(self, callback=None):
        """get a Channel instance"""
        if self.status == status.OPENED:
            ch_id = self._get_next_channel_id()
            ch = Channel(channel_id=ch_id, conn=self)
            self.channels[ch_id] = ch
            ch._open(callback)
            return ch
        else:
            raise AmqpStatusError('connection is not opened')

    def _handshake(self):
        self.stream.write('AMQP\x00\x00\x09\x01')
        FrameReader(self.stream, self._frame_loop)

    def _frame_loop(self, frame):
        if self.heartbeat:
            self.last_received_frame = time.time()
        self.channels[frame.channel].process_frame(frame)
        self._frame_count += 1
        if self.stream:
            # Every 5 frames ioloop gets the control back in order
            # to avoid hitting the recursion limit
            # reading one frame cost 13 levels of stack recursion
            # TODO check if always using _callbacks is faster that frame
            # counting
            if self._frame_count == 5:
                self._frame_count = 0
                cb = lambda: FrameReader(self.stream, self._frame_loop)
                self._add_ioloop_callback(cb)
            else:
                FrameReader(self.stream, self._frame_loop)

    if TORNADO_1_2:
        def _add_ioloop_callback(self, callback):
            self.io_loop._callbacks.append(callback)
    else:
        def _add_ioloop_callback(self, callback):
            self.io_loop._callbacks.add(callback)

    def close_stream(self):
        if self.stream is None:
            return

        try:
            self.stream.close()
        finally:
            self.stream = None

    def on_closed_stream(self):
        if self.status == status.CLOSED:
            return

        self.status = status.CLOSED
        if self.on_disconnect:
            try:
                self.on_disconnect()
            except Exception:
                logger.error('ERROR in on_disconnect() callback',
                             exc_info=True)

    def reset(self):
        for c in self.channels.values():
            if c is not self:
                c.reset()
        super(Connection, self).reset()
        self.close_stream()
class _HTTPConnection(object):
    _SUPPORTED_METHODS = set(
        ["GET", "HEAD", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"])

    def __init__(self, io_loop, client, request, release_callback,
                 final_callback, max_buffer_size, resolver):
        self.start_time = io_loop.time()
        self.io_loop = io_loop
        self.client = client
        self.request = request
        self.release_callback = release_callback
        self.final_callback = final_callback
        self.max_buffer_size = max_buffer_size
        self.resolver = resolver
        self.code = None
        self.headers = None
        self.chunks = None
        self._decompressor = None
        # Timeout handle returned by IOLoop.add_timeout
        self._timeout = None
        with stack_context.ExceptionStackContext(self._handle_exception):
            self.parsed = urlparse.urlsplit(_unicode(self.request.url))
            if self.parsed.scheme not in ("http", "https"):
                raise ValueError("Unsupported url scheme: %s" %
                                 self.request.url)
            # urlsplit results have hostname and port results, but they
            # didn't support ipv6 literals until python 2.7.
            netloc = self.parsed.netloc
            if "@" in netloc:
                userpass, _, netloc = netloc.rpartition("@")
            match = re.match(r'^(.+):(\d+)$', netloc)
            if match:
                host = match.group(1)
                port = int(match.group(2))
            else:
                host = netloc
                port = 443 if self.parsed.scheme == "https" else 80
            if re.match(r'^\[.*\]$', host):
                # raw ipv6 addresses in urls are enclosed in brackets
                host = host[1:-1]
            self.parsed_hostname = host  # save final host for _on_connect

            if request.allow_ipv6:
                af = socket.AF_UNSPEC
            else:
                # We only try the first IP we get from getaddrinfo,
                # so restrict to ipv4 by default.
                af = socket.AF_INET

            self.resolver.resolve(host, port, af, callback=self._on_resolve)

    def _on_resolve(self, addrinfo):
        af, sockaddr = addrinfo[0]

        if self.parsed.scheme == "https":
            ssl_options = {}
            if self.request.validate_cert:
                ssl_options["cert_reqs"] = ssl.CERT_REQUIRED
            if self.request.ca_certs is not None:
                ssl_options["ca_certs"] = self.request.ca_certs
            else:
                ssl_options["ca_certs"] = _DEFAULT_CA_CERTS
            if self.request.client_key is not None:
                ssl_options["keyfile"] = self.request.client_key
            if self.request.client_cert is not None:
                ssl_options["certfile"] = self.request.client_cert

            # SSL interoperability is tricky.  We want to disable
            # SSLv2 for security reasons; it wasn't disabled by default
            # until openssl 1.0.  The best way to do this is to use
            # the SSL_OP_NO_SSLv2, but that wasn't exposed to python
            # until 3.2.  Python 2.7 adds the ciphers argument, which
            # can also be used to disable SSLv2.  As a last resort
            # on python 2.6, we set ssl_version to SSLv3.  This is
            # more narrow than we'd like since it also breaks
            # compatibility with servers configured for TLSv1 only,
            # but nearly all servers support SSLv3:
            # http://blog.ivanristic.com/2011/09/ssl-survey-protocol-support.html
            if sys.version_info >= (2, 7):
                ssl_options["ciphers"] = "DEFAULT:!SSLv2"
            else:
                # This is really only necessary for pre-1.0 versions
                # of openssl, but python 2.6 doesn't expose version
                # information.
                ssl_options["ssl_version"] = ssl.PROTOCOL_SSLv3

            self.stream = SSLIOStream(socket.socket(af),
                                      io_loop=self.io_loop,
                                      ssl_options=ssl_options,
                                      max_buffer_size=self.max_buffer_size)
        else:
            self.stream = IOStream(socket.socket(af),
                                   io_loop=self.io_loop,
                                   max_buffer_size=self.max_buffer_size)
        timeout = min(self.request.connect_timeout,
                      self.request.request_timeout)
        if timeout:
            self._timeout = self.io_loop.add_timeout(
                self.start_time + timeout,
                stack_context.wrap(self._on_timeout))
        self.stream.set_close_callback(self._on_close)
        # ipv6 addresses are broken (in self.parsed.hostname) until
        # 2.7, here is correctly parsed value calculated in __init__
        self.stream.connect(sockaddr,
                            self._on_connect,
                            server_hostname=self.parsed_hostname)

    def _on_timeout(self):
        self._timeout = None
        if self.final_callback is not None:
            raise HTTPError(599, "Timeout")

    def _remove_timeout(self):
        if self._timeout is not None:
            self.io_loop.remove_timeout(self._timeout)
            self._timeout = None

    def _on_connect(self):
        self._remove_timeout()
        if self.request.request_timeout:
            self._timeout = self.io_loop.add_timeout(
                self.start_time + self.request.request_timeout,
                stack_context.wrap(self._on_timeout))
        if (self.request.method not in self._SUPPORTED_METHODS
                and not self.request.allow_nonstandard_methods):
            raise KeyError("unknown method %s" % self.request.method)
        for key in ('network_interface', 'proxy_host', 'proxy_port',
                    'proxy_username', 'proxy_password'):
            if getattr(self.request, key, None):
                raise NotImplementedError('%s not supported' % key)
        if "Connection" not in self.request.headers:
            self.request.headers["Connection"] = "close"
        if "Host" not in self.request.headers:
            if '@' in self.parsed.netloc:
                self.request.headers["Host"] = self.parsed.netloc.rpartition(
                    '@')[-1]
            else:
                self.request.headers["Host"] = self.parsed.netloc
        username, password = None, None
        if self.parsed.username is not None:
            username, password = self.parsed.username, self.parsed.password
        elif self.request.auth_username is not None:
            username = self.request.auth_username
            password = self.request.auth_password or ''
        if username is not None:
            auth = utf8(username) + b":" + utf8(password)
            self.request.headers["Authorization"] = (b"Basic " +
                                                     base64.b64encode(auth))
        if self.request.user_agent:
            self.request.headers["User-Agent"] = self.request.user_agent
        if not self.request.allow_nonstandard_methods:
            if self.request.method in ("POST", "PATCH", "PUT"):
                assert self.request.body is not None
            else:
                assert self.request.body is None
        if self.request.body is not None:
            self.request.headers["Content-Length"] = str(len(
                self.request.body))
        if (self.request.method == "POST"
                and "Content-Type" not in self.request.headers):
            self.request.headers[
                "Content-Type"] = "application/x-www-form-urlencoded"
        if self.request.use_gzip:
            self.request.headers["Accept-Encoding"] = "gzip"
        req_path = ((self.parsed.path or '/') +
                    (('?' + self.parsed.query) if self.parsed.query else ''))
        request_lines = [
            utf8("%s %s HTTP/1.1" % (self.request.method, req_path))
        ]
        for k, v in self.request.headers.get_all():
            line = utf8(k) + b": " + utf8(v)
            if b'\n' in line:
                raise ValueError('Newline in header: ' + repr(line))
            request_lines.append(line)
        self.stream.write(b"\r\n".join(request_lines) + b"\r\n\r\n")
        if self.request.body is not None:
            self.stream.write(self.request.body)
        self.stream.read_until_regex(b"\r?\n\r?\n", self._on_headers)

    def _release(self):
        if self.release_callback is not None:
            release_callback = self.release_callback
            self.release_callback = None
            release_callback()

    def _run_callback(self, response):
        self._release()
        if self.final_callback is not None:
            final_callback = self.final_callback
            self.final_callback = None
            self.io_loop.add_callback(final_callback, response)

    def _handle_exception(self, typ, value, tb):
        if self.final_callback:
            self._remove_timeout()
            gen_log.warning("uncaught exception", exc_info=(typ, value, tb))
            self._run_callback(
                HTTPResponse(
                    self.request,
                    599,
                    error=value,
                    request_time=self.io_loop.time() - self.start_time,
                ))

            if hasattr(self, "stream"):
                self.stream.close()
            return True
        else:
            # If our callback has already been called, we are probably
            # catching an exception that is not caused by us but rather
            # some child of our callback. Rather than drop it on the floor,
            # pass it along.
            return False

    def _on_close(self):
        if self.final_callback is not None:
            message = "Connection closed"
            if self.stream.error:
                message = str(self.stream.error)
            raise HTTPError(599, message)

    def _handle_1xx(self, code):
        self.stream.read_until_regex(b"\r?\n\r?\n", self._on_headers)

    def _on_headers(self, data):
        data = native_str(data.decode("latin1"))
        first_line, _, header_data = data.partition("\n")
        match = re.match("HTTP/1.[01] ([0-9]+) ([^\r]*)", first_line)
        assert match
        code = int(match.group(1))
        self.headers = HTTPHeaders.parse(header_data)
        if 100 <= code < 200:
            self._handle_1xx(code)
            return
        else:
            self.code = code
            self.reason = match.group(2)

        if "Content-Length" in self.headers:
            if "," in self.headers["Content-Length"]:
                # Proxies sometimes cause Content-Length headers to get
                # duplicated.  If all the values are identical then we can
                # use them but if they differ it's an error.
                pieces = re.split(r',\s*', self.headers["Content-Length"])
                if any(i != pieces[0] for i in pieces):
                    raise ValueError("Multiple unequal Content-Lengths: %r" %
                                     self.headers["Content-Length"])
                self.headers["Content-Length"] = pieces[0]
            content_length = int(self.headers["Content-Length"])
        else:
            content_length = None

        if self.request.header_callback is not None:
            # re-attach the newline we split on earlier
            self.request.header_callback(first_line + _)
            for k, v in self.headers.get_all():
                self.request.header_callback("%s: %s\r\n" % (k, v))
            self.request.header_callback('\r\n')

        if self.request.method == "HEAD" or self.code == 304:
            # HEAD requests and 304 responses never have content, even
            # though they may have content-length headers
            self._on_body(b"")
            return
        if 100 <= self.code < 200 or self.code == 204:
            # These response codes never have bodies
            # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3
            if ("Transfer-Encoding" in self.headers
                    or content_length not in (None, 0)):
                raise ValueError("Response with code %d should not have body" %
                                 self.code)
            self._on_body(b"")
            return

        if (self.request.use_gzip
                and self.headers.get("Content-Encoding") == "gzip"):
            self._decompressor = GzipDecompressor()
        if self.headers.get("Transfer-Encoding") == "chunked":
            self.chunks = []
            self.stream.read_until(b"\r\n", self._on_chunk_length)
        elif content_length is not None:
            self.stream.read_bytes(content_length, self._on_body)
        else:
            self.stream.read_until_close(self._on_body)

    def _on_body(self, data):
        self._remove_timeout()
        original_request = getattr(self.request, "original_request",
                                   self.request)
        if (self.request.follow_redirects and self.request.max_redirects > 0
                and self.code in (301, 302, 303, 307)):
            assert isinstance(self.request, _RequestProxy)
            new_request = copy.copy(self.request.request)
            new_request.url = urlparse.urljoin(self.request.url,
                                               self.headers["Location"])
            new_request.max_redirects = self.request.max_redirects - 1
            del new_request.headers["Host"]
            # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4
            # Client SHOULD make a GET request after a 303.
            # According to the spec, 302 should be followed by the same
            # method as the original request, but in practice browsers
            # treat 302 the same as 303, and many servers use 302 for
            # compatibility with pre-HTTP/1.1 user agents which don't
            # understand the 303 status.
            if self.code in (302, 303):
                new_request.method = "GET"
                new_request.body = None
                for h in [
                        "Content-Length", "Content-Type", "Content-Encoding",
                        "Transfer-Encoding"
                ]:
                    try:
                        del self.request.headers[h]
                    except KeyError:
                        pass
            new_request.original_request = original_request
            final_callback = self.final_callback
            self.final_callback = None
            self._release()
            self.client.fetch(new_request, final_callback)
            self.stream.close()
            return
        if self._decompressor:
            data = (self._decompressor.decompress(data) +
                    self._decompressor.flush())
        if self.request.streaming_callback:
            if self.chunks is None:
                # if chunks is not None, we already called streaming_callback
                # in _on_chunk_data
                self.request.streaming_callback(data)
            buffer = BytesIO()
        else:
            buffer = BytesIO(data)  # TODO: don't require one big string?
        response = HTTPResponse(original_request,
                                self.code,
                                reason=self.reason,
                                headers=self.headers,
                                request_time=self.io_loop.time() -
                                self.start_time,
                                buffer=buffer,
                                effective_url=self.request.url)
        self._run_callback(response)
        self.stream.close()

    def _on_chunk_length(self, data):
        # TODO: "chunk extensions" http://tools.ietf.org/html/rfc2616#section-3.6.1
        length = int(data.strip(), 16)
        if length == 0:
            if self._decompressor is not None:
                tail = self._decompressor.flush()
                if tail:
                    # I believe the tail will always be empty (i.e.
                    # decompress will return all it can).  The purpose
                    # of the flush call is to detect errors such
                    # as truncated input.  But in case it ever returns
                    # anything, treat it as an extra chunk
                    if self.request.streaming_callback is not None:
                        self.request.streaming_callback(tail)
                    else:
                        self.chunks.append(tail)
                # all the data has been decompressed, so we don't need to
                # decompress again in _on_body
                self._decompressor = None
            self._on_body(b''.join(self.chunks))
        else:
            self.stream.read_bytes(
                length + 2,  # chunk ends with \r\n
                self._on_chunk_data)

    def _on_chunk_data(self, data):
        assert data[-2:] == b"\r\n"
        chunk = data[:-2]
        if self._decompressor:
            chunk = self._decompressor.decompress(chunk)
        if self.request.streaming_callback is not None:
            self.request.streaming_callback(chunk)
        else:
            self.chunks.append(chunk)
        self.stream.read_until(b"\r\n", self._on_chunk_length)
Ejemplo n.º 54
0
class PushMessageReceiver(object):
    def __init__(self):
        self.connected = False
        self._stream = None
        self.address = None
        self.family = None
        self.delayed_call = None
        self._registered_handlers = {}

    def register_message_handler(self, message_type, handler):
        self._registered_handlers[message_type] = handler

    def unregister_message_handler(self, message_type):
        try:
            del self._registered_handlers[message_type]
        except KeyError:
            pass

    def unregister_all(self):
        self._registered_handlers = {}

    def connect(self, address, family=socket.AF_INET, retry_interval=1):
        self.address = address
        self.family = family
        self.retry_interval = retry_interval
        self._connect()

    def _connect(self):
        if self.connected:
            if self.delayed_call:
                IOLoop.current().remove_timeout(self.delayed_call)
                self.delayed_call = None
            else:
                raise RuntimeError("Already connected")
        else:
            self.delayed_call = IOLoop.current().call_later(1, self._connect)
            stream_socket = socket.socket(family=self.family)
            self._stream = IOStream(stream_socket)
            self._stream.connect(self.address, self._on_connect)

    def _on_connect(self):
        self.connected = True
        self._stream.set_close_callback(self._on_closed)
        self._stream.read_until('\n', self._handle_greeting)

    def _on_closed(self):
        if self.connected:
            # The other side closed us, try to reconnect
            self.connected = False
            self.connect(self.address, self.family)

    def _handle_greeting(self, greetings):
        self._stream.read_until('\n', self._handle_message)

    def _handle_message(self, raw_message):
        message = json.loads(raw_message.strip())
        try:
            handler = self._registered_handlers[message['type']]
            handler(message['content'])
        except KeyError:
            pass
        finally:
            # continue getting messages as long as we are connected
            if self.connected:
                self._stream.read_until('\n', self._handle_message)

    def close(self):
        if self._stream:
            self.connected = False
            self._stream.close()
            self._stream = None
class _HTTPConnection(object):
    _SUPPORTED_METHODS = set(["GET", "HEAD", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"])

    def __init__(self, io_loop, client, request, release_callback,
                 final_callback, max_buffer_size):
        self.start_time = time.time()
        self.io_loop = io_loop
        self.client = client
        self.request = request
        self.release_callback = release_callback
        self.final_callback = final_callback
        self.code = None
        self.headers = None
        self.chunks = None
        self._decompressor = None
        # Timeout handle returned by IOLoop.add_timeout
        self._timeout = None
        with stack_context.StackContext(self.cleanup):
            parsed = urlparse.urlsplit(_unicode(self.request.url))
            if ssl is None and parsed.scheme == "https":
                raise ValueError("HTTPS requires either python2.6+ or "
                                 "curl_httpclient")
            if parsed.scheme not in ("http", "https"):
                raise ValueError("Unsupported url scheme: %s" %
                                 self.request.url)
            # urlsplit results have hostname and port results, but they
            # didn't support ipv6 literals until python 2.7.
            netloc = parsed.netloc
            if "@" in netloc:
                userpass, _, netloc = netloc.rpartition("@")
            match = re.match(r'^(.+):(\d+)$', netloc)
            if match:
                host = match.group(1)
                port = int(match.group(2))
            else:
                host = netloc
                port = 443 if parsed.scheme == "https" else 80
            if re.match(r'^\[.*\]$', host):
                # raw ipv6 addresses in urls are enclosed in brackets
                host = host[1:-1]
            parsed_hostname = host  # save final parsed host for _on_connect
            if self.client.hostname_mapping is not None:
                host = self.client.hostname_mapping.get(host, host)

            if request.allow_ipv6:
                af = socket.AF_UNSPEC
            else:
                # We only try the first IP we get from getaddrinfo,
                # so restrict to ipv4 by default.
                af = socket.AF_INET

            addrinfo = socket.getaddrinfo(host, port, af, socket.SOCK_STREAM,
                                          0, 0)
            af, socktype, proto, canonname, sockaddr = addrinfo[0]

            if parsed.scheme == "https":
                ssl_options = {}
                if request.validate_cert:
                    ssl_options["cert_reqs"] = ssl.CERT_REQUIRED
                if request.ca_certs is not None:
                    ssl_options["ca_certs"] = request.ca_certs
                else:
                    ssl_options["ca_certs"] = _DEFAULT_CA_CERTS
                if request.client_key is not None:
                    ssl_options["keyfile"] = request.client_key
                if request.client_cert is not None:
                    ssl_options["certfile"] = request.client_cert

                # SSL interoperability is tricky.  We want to disable
                # SSLv2 for security reasons; it wasn't disabled by default
                # until openssl 1.0.  The best way to do this is to use
                # the SSL_OP_NO_SSLv2, but that wasn't exposed to python
                # until 3.2.  Python 2.7 adds the ciphers argument, which
                # can also be used to disable SSLv2.  As a last resort
                # on python 2.6, we set ssl_version to SSLv3.  This is
                # more narrow than we'd like since it also breaks
                # compatibility with servers configured for TLSv1 only,
                # but nearly all servers support SSLv3:
                # http://blog.ivanristic.com/2011/09/ssl-survey-protocol-support.html
                if sys.version_info >= (2, 7):
                    ssl_options["ciphers"] = "DEFAULT:!SSLv2"
                else:
                    # This is really only necessary for pre-1.0 versions
                    # of openssl, but python 2.6 doesn't expose version
                    # information.
                    ssl_options["ssl_version"] = ssl.PROTOCOL_SSLv3

                self.stream = SSLIOStream(socket.socket(af, socktype, proto),
                                          io_loop=self.io_loop,
                                          ssl_options=ssl_options,
                                          max_buffer_size=max_buffer_size)
            else:
                self.stream = IOStream(socket.socket(af, socktype, proto),
                                       io_loop=self.io_loop,
                                       max_buffer_size=max_buffer_size)
            timeout = min(request.connect_timeout, request.request_timeout)
            if timeout:
                self._timeout = self.io_loop.add_timeout(
                    self.start_time + timeout,
                    self._on_timeout)
            self.stream.set_close_callback(self._on_close)
            self.stream.connect(sockaddr,
                                functools.partial(self._on_connect, parsed,
                                                  parsed_hostname))

    def _on_timeout(self):
        self._timeout = None
        self._run_callback(HTTPResponse(self.request, 599,
                                        request_time=time.time() - self.start_time,
                                        error=HTTPError(599, "Timeout")))
        self.stream.close()

    def _on_connect(self, parsed, parsed_hostname):
        if self._timeout is not None:
            self.io_loop.remove_timeout(self._timeout)
            self._timeout = None
        if self.request.request_timeout:
            self._timeout = self.io_loop.add_timeout(
                self.start_time + self.request.request_timeout,
                self._on_timeout)
        if (self.request.validate_cert and
            isinstance(self.stream, SSLIOStream)):
            match_hostname(self.stream.socket.getpeercert(),
                           # ipv6 addresses are broken (in
                           # parsed.hostname) until 2.7, here is
                           # correctly parsed value calculated in
                           # __init__
                           parsed_hostname)
        if (self.request.method not in self._SUPPORTED_METHODS and
            not self.request.allow_nonstandard_methods):
            raise KeyError("unknown method %s" % self.request.method)
        for key in ('network_interface',
                    'proxy_host', 'proxy_port',
                    'proxy_username', 'proxy_password'):
            if getattr(self.request, key, None):
                raise NotImplementedError('%s not supported' % key)
        if "Connection" not in self.request.headers:
            self.request.headers["Connection"] = "close"
        if "Host" not in self.request.headers:
            if '@' in parsed.netloc:
                self.request.headers["Host"] = parsed.netloc.rpartition('@')[-1]
            else:
                self.request.headers["Host"] = parsed.netloc
        username, password = None, None
        if parsed.username is not None:
            username, password = parsed.username, parsed.password
        elif self.request.auth_username is not None:
            username = self.request.auth_username
            password = self.request.auth_password or ''
        if username is not None:
            auth = utf8(username) + b(":") + utf8(password)
            self.request.headers["Authorization"] = (b("Basic ") +
                                                     base64.b64encode(auth))
        if self.request.user_agent:
            self.request.headers["User-Agent"] = self.request.user_agent
        if not self.request.allow_nonstandard_methods:
            if self.request.method in ("POST", "PATCH", "PUT"):
                assert self.request.body is not None
            else:
                assert self.request.body is None
        if self.request.body is not None:
            self.request.headers["Content-Length"] = str(len(
                    self.request.body))
        if (self.request.method == "POST" and
            "Content-Type" not in self.request.headers):
            self.request.headers["Content-Type"] = "application/x-www-form-urlencoded"
        if self.request.use_gzip:
            self.request.headers["Accept-Encoding"] = "gzip"
        req_path = ((parsed.path or '/') +
                (('?' + parsed.query) if parsed.query else ''))
        request_lines = [utf8("%s %s HTTP/1.1" % (self.request.method,
                                                  req_path))]
        for k, v in self.request.headers.get_all():
            line = utf8(k) + b(": ") + utf8(v)
            if b('\n') in line:
                raise ValueError('Newline in header: ' + repr(line))
            request_lines.append(line)
        self.stream.write(b("\r\n").join(request_lines) + b("\r\n\r\n"))
        if self.request.body is not None:
            self.stream.write(self.request.body)
        self.stream.read_until_regex(b("\r?\n\r?\n"), self._on_headers)

    def _release(self):
        if self.release_callback is not None:
            release_callback = self.release_callback
            self.release_callback = None
            release_callback()

    def _run_callback(self, response):
        self._release()
        if self.final_callback is not None:
            final_callback = self.final_callback
            self.final_callback = None
            final_callback(response)

    @contextlib.contextmanager
    def cleanup(self):
        try:
            yield
        except Exception, e:
            #logging.warning("uncaught exception", exc_info=True)
            self._run_callback(HTTPResponse(self.request, 599, error=e,
                                request_time=time.time() - self.start_time,
                                ))
            if hasattr(self, "stream"):
                self.stream.close()
Ejemplo n.º 56
0
class Connection(object):
    def __init__(self,
                 host='localhost',
                 port=6379,
                 unix_socket_path=None,
                 event_handler_proxy=None,
                 stop_after=None,
                 io_loop=None):
        self.host = host
        self.port = port
        self.unix_socket_path = unix_socket_path
        self._event_handler = event_handler_proxy
        self.timeout = stop_after
        self._stream = None
        self._io_loop = io_loop

        self.in_progress = False
        self.read_callbacks = set()
        self.ready_callbacks = deque()
        self._lock = 0
        self.info = {'db': 0, 'pass': None}

    def __del__(self):
        self.disconnect()

    def execute_pending_command(self):
        # Continue with the pending command execution
        # if all read operations are completed.
        if not self.read_callbacks and self.ready_callbacks:
            # Pop a SINGLE callback from the queue and execute it.
            # The next one will be executed from the code
            # invoked by the callback
            callback = self.ready_callbacks.popleft()
            callback()

    def ready(self):
        return (not self.read_callbacks and not self.ready_callbacks)

    def wait_until_ready(self, callback=None):
        if callback:
            if not self.ready():
                callback = stack_context.wrap(callback)
                self.ready_callbacks.append(callback)
            else:
                callback()

    def connect(self):
        if not self._stream:
            try:
                if self.unix_socket_path:
                    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
                    sock.settimeout(self.timeout)
                    sock.connect(self.unix_socket_path)
                else:
                    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
                    sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
                    sock.settimeout(self.timeout)
                    sock.connect((self.host, self.port))
                self._stream = IOStream(sock, io_loop=self._io_loop)
                self._stream.set_close_callback(self.on_stream_close)
                self.info['db'] = 0
                self.info['pass'] = None
            except socket.error as e:
                raise ConnectionError(str(e))
            self.fire_event('on_connect')

    def on_stream_close(self):
        if self._stream:
            self.disconnect()
            callbacks = self.read_callbacks
            self.read_callbacks = set()
            for callback in callbacks:
                callback()

    def disconnect(self):
        if self._stream:
            s = self._stream
            self._stream = None
            try:
                if s.socket:
                    s.socket.shutdown(socket.SHUT_RDWR)
                s.close()
            except:
                pass

    def fire_event(self, event):
        event_handler = self._event_handler
        if event_handler:
            try:
                getattr(event_handler, event)()
            except AttributeError:
                pass

    def write(self, data, callback=None):
        if not self._stream:
            raise ConnectionError('Tried to write to '
                                  'non-existent connection')

        if callback:
            callback = stack_context.wrap(callback)
            _callback = lambda: callback(None)
            self.read_callbacks.add(_callback)
            cb = partial(self.read_callback, _callback)
        else:
            cb = None
        try:
            if PY3:
                data = bytes(data, encoding='utf-8')
            self._stream.write(data, callback=cb)
        except IOError as e:
            self.disconnect()
            raise ConnectionError(e.message)

    def read(self, length, callback=None):
        try:
            if not self._stream:
                self.disconnect()
                raise ConnectionError('Tried to read from '
                                      'non-existent connection')
            callback = stack_context.wrap(callback)
            self.read_callbacks.add(callback)
            self._stream.read_bytes(length,
                                    callback=partial(self.read_callback,
                                                     callback))
        except IOError:
            self.fire_event('on_disconnect')

    def read_callback(self, callback, *args, **kwargs):
        try:
            self.read_callbacks.remove(callback)
        except KeyError:
            pass
        callback(*args, **kwargs)

    def readline(self, callback=None):
        try:
            if not self._stream:
                self.disconnect()
                raise ConnectionError('Tried to read from '
                                      'non-existent connection')
            callback = stack_context.wrap(callback)
            self.read_callbacks.add(callback)
            callback = partial(self.read_callback, callback)
            self._stream.read_until(CRLF, callback=callback)
        except IOError:
            self.fire_event('on_disconnect')

    def connected(self):
        if self._stream:
            return True
        return False
Ejemplo n.º 57
0
class TornadoClient(Client):
    """A non-blocking Pomelo client by tornado ioloop

    Usage :

        class ClientHandler(object) :

            def on_recv_data(self, client, proto_type, data) :
                print "recv_data..."
                return data

            def on_connected(self, client, user_data) :
                print "connect..."
                client.send_heartbeat()

            def on_disconnect(self, client) :
                print "disconnect..."

            def on_heartbeat(self, client) :
                print "heartbeat..."
                send request ...

            def on_response(self, client, route, request, response) :
                print "response..."

            def on_push(self, client, route, push_data) :
                print "notify..."

        handler = ClientHandler()
        client = TornadoClient(handler)
        client.connect(host, int(port))
        client.run()
        tornado.ioloop.IOLoop.current().start()
    """
    def __init__(self, handler):
        self.socket = socket(AF_INET, SOCK_STREAM)
        self.iostream = None
        self.protocol_package = None
        super(TornadoClient, self).__init__(handler)

    def connect(self, host, port):
        self.iostream = IOStream(self.socket)
        self.iostream.set_close_callback(self.on_close)
        self.iostream.connect((host, port), self.on_connect)

    def on_connect(self):
        self.send_sync()
        self.on_data()

    def on_close(self):
        if hasattr(self.handler, 'on_disconnect'):
            self.handler.on_disconnect(self)

    def send(self, data):
        assert not self.iostream.closed(), "iostream has closed"
        if not isinstance(data, bytes):
            data = bytes(data)
        self.iostream.write(data)

    def on_data(self):
        assert not self.iostream.closed(), "iostream has closed"
        if None is self.protocol_package or self.protocol_package.completed():
            self.iostream.read_bytes(4, self.on_head)

    def on_head(self, head):
        self.protocol_package = Protocol.unpack(head)
        self.iostream.read_bytes(self.protocol_package.length, self.on_body)

    def on_body(self, body):
        if hasattr(self.handler, 'on_recv_data'):
            body = self.handler.on_recv_data(self,
                                             self.protocol_package.proto_type,
                                             body)
        self.protocol_package.append(body)
        self.on_protocol(self.protocol_package)
        self.on_data()

    def close(self):
        if self.iostream:
            self.iostream.close()
Ejemplo n.º 58
0
class _HTTPConnection(object):
    _SUPPORTED_METHODS = set(["GET", "HEAD", "POST", "PUT", "DELETE"])

    def __init__(self, io_loop, client, request, release_callback,
                 final_callback, max_buffer_size):
        self.start_time = time.time()
        self.io_loop = io_loop
        self.client = client
        self.request = request
        self.release_callback = release_callback
        self.final_callback = final_callback
        self.code = None
        self.headers = None
        self.chunks = None
        self._decompressor = None
        # Timeout handle returned by IOLoop.add_timeout
        self._timeout = None
        with stack_context.StackContext(self.cleanup):
            parsed = urlparse.urlsplit(_unicode(self.request.url))
            if ssl is None and parsed.scheme == "https":
                raise ValueError("HTTPS requires either python2.6+ or "
                                 "curl_httpclient")
            if parsed.scheme not in ("http", "https"):
                raise ValueError("Unsupported url scheme: %s" %
                                 self.request.url)
            # urlsplit results have hostname and port results, but they
            # didn't support ipv6 literals until python 2.7.
            netloc = parsed.netloc
            if "@" in netloc:
                userpass, _, netloc = netloc.rpartition("@")
            match = re.match(r'^(.+):(\d+)$', netloc)
            if match:
                host = match.group(1)
                port = int(match.group(2))
            else:
                host = netloc
                port = 443 if parsed.scheme == "https" else 80
            if re.match(r'^\[.*\]$', host):
                # raw ipv6 addresses in urls are enclosed in brackets
                host = host[1:-1]
            if self.client.hostname_mapping is not None:
                host = self.client.hostname_mapping.get(host, host)

            if request.allow_ipv6:
                af = socket.AF_UNSPEC
            else:
                # We only try the first IP we get from getaddrinfo,
                # so restrict to ipv4 by default.
                af = socket.AF_INET

            addrinfo = socket.getaddrinfo(host, port, af, socket.SOCK_STREAM,
                                          0, 0)
            af, socktype, proto, canonname, sockaddr = addrinfo[0]

            if parsed.scheme == "https":
                ssl_options = {}
                if request.validate_cert:
                    ssl_options["cert_reqs"] = ssl.CERT_REQUIRED
                if request.ca_certs is not None:
                    ssl_options["ca_certs"] = request.ca_certs
                else:
                    ssl_options["ca_certs"] = _DEFAULT_CA_CERTS
                if request.client_key is not None:
                    ssl_options["keyfile"] = request.client_key
                if request.client_cert is not None:
                    ssl_options["certfile"] = request.client_cert
                self.stream = SSLIOStream(socket.socket(af, socktype, proto),
                                          io_loop=self.io_loop,
                                          ssl_options=ssl_options,
                                          max_buffer_size=max_buffer_size)
            else:
                self.stream = IOStream(socket.socket(af, socktype, proto),
                                       io_loop=self.io_loop,
                                       max_buffer_size=max_buffer_size)
            timeout = min(request.connect_timeout, request.request_timeout)
            if timeout:
                self._timeout = self.io_loop.add_timeout(
                    self.start_time + timeout,
                    self._on_timeout)
            self.stream.set_close_callback(self._on_close)
            self.stream.connect(sockaddr,
                                functools.partial(self._on_connect, parsed))

    def _on_timeout(self):
        self._timeout = None
        self._run_callback(HTTPResponse(self.request, 599,
                                        request_time=time.time() - self.start_time,
                                        error=HTTPError(599, "Timeout")))
        self.stream.close()

    def _on_connect(self, parsed):
        if self._timeout is not None:
            self.io_loop.remove_timeout(self._timeout)
            self._timeout = None
        if self.request.request_timeout:
            self._timeout = self.io_loop.add_timeout(
                self.start_time + self.request.request_timeout,
                self._on_timeout)
        if (self.request.validate_cert and
            isinstance(self.stream, SSLIOStream)):
            match_hostname(self.stream.socket.getpeercert(),
                           parsed.hostname)
        if (self.request.method not in self._SUPPORTED_METHODS and
            not self.request.allow_nonstandard_methods):
            raise KeyError("unknown method %s" % self.request.method)
        for key in ('network_interface',
                    'proxy_host', 'proxy_port',
                    'proxy_username', 'proxy_password'):
            if getattr(self.request, key, None):
                raise NotImplementedError('%s not supported' % key)
        if "Host" not in self.request.headers:
            self.request.headers["Host"] = parsed.netloc
        username, password = None, None
        if parsed.username is not None:
            username, password = parsed.username, parsed.password
        elif self.request.auth_username is not None:
            username = self.request.auth_username
            password = self.request.auth_password
        if username is not None:
            auth = utf8(username) + b(":") + utf8(password)
            self.request.headers["Authorization"] = (b("Basic ") +
                                                     base64.b64encode(auth))
        if self.request.user_agent:
            self.request.headers["User-Agent"] = self.request.user_agent
        if not self.request.allow_nonstandard_methods:
            if self.request.method in ("POST", "PUT"):
                assert self.request.body is not None
            else:
                assert self.request.body is None
        if self.request.body is not None:
            self.request.headers["Content-Length"] = str(len(
                    self.request.body))
        if (self.request.method == "POST" and
            "Content-Type" not in self.request.headers):
            self.request.headers["Content-Type"] = "application/x-www-form-urlencoded"
        if self.request.use_gzip:
            self.request.headers["Accept-Encoding"] = "gzip"
        req_path = ((parsed.path or '/') +
                (('?' + parsed.query) if parsed.query else ''))
        request_lines = [utf8("%s %s HTTP/1.1" % (self.request.method,
                                                  req_path))]
        for k, v in self.request.headers.get_all():
            line = utf8(k) + b(": ") + utf8(v)
            if b('\n') in line:
                raise ValueError('Newline in header: ' + repr(line))
            request_lines.append(line)
        self.stream.write(b("\r\n").join(request_lines) + b("\r\n\r\n"))
        if self.request.body is not None:
            self.stream.write(self.request.body)
        self.stream.read_until_regex(b("\r?\n\r?\n"), self._on_headers)

    def _release(self):
        if self.release_callback is not None:
            release_callback = self.release_callback
            self.release_callback = None
            release_callback()

    def _run_callback(self, response):
        self._release()
        if self.final_callback is not None:
            final_callback = self.final_callback
            self.final_callback = None
            final_callback(response)

    @contextlib.contextmanager
    def cleanup(self):
        try:
            yield
        except Exception, e:
            logging.warning("uncaught exception", exc_info=True)
            self._run_callback(HTTPResponse(self.request, 599, error=e, 
                                request_time=time.time() - self.start_time,
                                ))
Ejemplo n.º 59
0
class _HTTPConnection(object):
    _SUPPORTED_METHODS = set(["GET", "HEAD", "POST", "PUT", "DELETE"])

    def __init__(self, io_loop, client, request, callback):
        self.start_time = time.time()
        self.io_loop = io_loop
        self.client = client
        self.request = request
        self.callback = callback
        self.code = None
        self.headers = None
        self.chunks = None
        self._decompressor = None
        # Timeout handle returned by IOLoop.add_timeout
        self._timeout = None
        with stack_context.StackContext(self.cleanup):
            parsed = urlparse.urlsplit(self.request.url)
            host = parsed.hostname
            if parsed.port is None:
                port = 443 if parsed.scheme == "https" else 80
            else:
                port = parsed.port
            if self.client.hostname_mapping is not None:
                host = self.client.hostname_mapping.get(host, host)

            if parsed.scheme == "https":
                ssl_options = {}
                if request.validate_cert:
                    ssl_options["cert_reqs"] = ssl.CERT_REQUIRED
                if request.ca_certs is not None:
                    ssl_options["ca_certs"] = request.ca_certs
                else:
                    ssl_options["ca_certs"] = _DEFAULT_CA_CERTS
                self.stream = SSLIOStream(socket.socket(),
                                          io_loop=self.io_loop,
                                          ssl_options=ssl_options)
            else:
                self.stream = IOStream(socket.socket(), io_loop=self.io_loop)
            timeout = min(request.connect_timeout, request.request_timeout)
            if timeout:
                self._connect_timeout = self.io_loop.add_timeout(
                    self.start_time + timeout, self._on_timeout)
            self.stream.set_close_callback(self._on_close)
            self.stream.connect((host, port),
                                functools.partial(self._on_connect, parsed))

    def _on_timeout(self):
        self._timeout = None
        if self.callback is not None:
            self.callback(
                HTTPResponse(self.request,
                             599,
                             error=HTTPError(599, "Timeout")))
            self.callback = None
        self.stream.close()

    def _on_connect(self, parsed):
        if self._timeout is not None:
            self.io_loop.remove_callback(self._timeout)
            self._timeout = None
        if self.request.request_timeout:
            self._timeout = self.io_loop.add_timeout(
                self.start_time + self.request.request_timeout,
                self._on_timeout)
        if (self.request.validate_cert
                and isinstance(self.stream, SSLIOStream)):
            match_hostname(self.stream.socket.getpeercert(), parsed.hostname)
        if (self.request.method not in self._SUPPORTED_METHODS
                and not self.request.allow_nonstandard_methods):
            raise KeyError("unknown method %s" % self.request.method)
        for key in ('network_interface', 'proxy_host', 'proxy_port',
                    'proxy_username', 'proxy_password'):
            if getattr(self.request, key, None):
                raise NotImplementedError('%s not supported' % key)
        if "Host" not in self.request.headers:
            self.request.headers["Host"] = parsed.netloc
        username, password = None, None
        if parsed.username is not None:
            username, password = parsed.username, parsed.password
        elif self.request.auth_username is not None:
            username = self.request.auth_username
            password = self.request.auth_password
        if username is not None:
            auth = "%s:%s" % (username, password)
            self.request.headers["Authorization"] = ("Basic %s" %
                                                     base64.b64encode(auth))
        if self.request.user_agent:
            self.request.headers["User-Agent"] = self.request.user_agent
        has_body = self.request.method in ("POST", "PUT")
        if has_body:
            assert self.request.body is not None
            self.request.headers["Content-Length"] = len(self.request.body)
        else:
            assert self.request.body is None
        if (self.request.method == "POST"
                and "Content-Type" not in self.request.headers):
            self.request.headers[
                "Content-Type"] = "application/x-www-form-urlencoded"
        if self.request.use_gzip:
            self.request.headers["Accept-Encoding"] = "gzip"
        req_path = ((parsed.path or '/') +
                    (('?' + parsed.query) if parsed.query else ''))
        request_lines = ["%s %s HTTP/1.1" % (self.request.method, req_path)]
        for k, v in self.request.headers.get_all():
            line = "%s: %s" % (k, v)
            if '\n' in line:
                raise ValueError('Newline in header: ' + repr(line))
            request_lines.append(line)
        self.stream.write("\r\n".join(request_lines) + "\r\n\r\n")
        if has_body:
            self.stream.write(self.request.body)
        self.stream.read_until("\r\n\r\n", self._on_headers)

    @contextlib.contextmanager
    def cleanup(self):
        try:
            yield
        except Exception, e:
            logging.warning("uncaught exception", exc_info=True)
            if self.callback is not None:
                callback = self.callback
                self.callback = None
                callback(HTTPResponse(self.request, 599, error=e))