def _open_channel(self, run): assert self.connection if not self._is_run_id_tracked(run): log.warning("Run %s no longer tracked (_open_channel)", run.id) return assert self.run_states[run.id].state < RUN_STATE_RUNNING self.run_states[run.id].state = RUN_STATE_STARTING chan = ssh.ExecChannel(conn=self.connection) chan.addOutputCallback(run.write_stdout) chan.addErrorCallback(run.write_stderr) chan.addEndCallback(run.done) chan.command = run.command chan.start_defer = defer.Deferred() chan.start_defer.addCallback(self._run_started, run) chan.start_defer.addErrback(self._run_start_error, run) chan.exit_defer = defer.Deferred() chan.exit_defer.addCallback(self._channel_complete, run) chan.exit_defer.addErrback(self._channel_complete_unknown, run) twistedutils.defer_timeout(chan.start_defer, RUN_START_TIMEOUT) self.run_states[run.id].channel = chan # TODO: I believe this needs to be checking the health of the connection # before trying to open a new channel. If the connection is gone it # needs to re-establish, or if the connection is not responding # we shouldn't create this new channel self.connection.openChannel(chan)
def _service_stopped(self, connection): """Called when the SSH service has disconnected fully. We should be in a state where we know there are no runs in progress because all the SSH channels should have disconnected them. """ assert self.connection is connection self.connection = None log.info("Service to %s stopped", self.hostname) for run_id, run in self.run_states.iteritems(): if run.state == RUN_STATE_CONNECTING: # Now we can trigger a reconnect and re-start any waiting runs. self._connect_then_run(run) elif run.state == RUN_STATE_RUNNING: self._fail_run(run, None) elif run.state == RUN_STATE_STARTING: if run.channel and run.channel.start_defer is not None: # This means our run IS still waiting to start. There # should be an outstanding timeout sitting on this guy as # well. We'll just short circut it. twistedutils.defer_timeout(run.channel.start_defer, 0) else: # Doesn't seem like this should ever happen. log.warning("Run %r caught in starting state, but" " start_defer is over.", run_id) self._fail_run(run, None) else: # Service ended. The open channels should know how to handle # this (and cleanup) themselves, so if there should not be any # runs except those waiting to connect raise Error("Run %s in state %s when service stopped", run_id, run.state)
def _open_channel(self, run): assert self.connection assert self.run_states[run.id].state < RUN_STATE_RUNNING self.run_states[run.id].state = RUN_STATE_STARTING chan = ssh.ExecChannel(conn=self.connection) chan.addOutputCallback(run.write_stdout) chan.addErrorCallback(run.write_stderr) chan.addEndCallback(run.done) chan.command = run.command chan.start_defer = defer.Deferred() chan.start_defer.addCallback(self._run_started, run) chan.start_defer.addErrback(self._run_start_error, run) chan.exit_defer = defer.Deferred() chan.exit_defer.addCallback(self._channel_complete, run) chan.exit_defer.addErrback(self._channel_complete_unknown, run) twistedutils.defer_timeout(chan.start_defer, RUN_START_TIMEOUT) self.run_states[run.id].channel = chan self.connection.openChannel(chan)
def _connect(self): # This is complicated because we have to deal with a few different # steps before our connection is really available for us: # 1. Transport is created (our client creator does this) # 2. Our transport is secure, and we can create our connection # 3. The connection service is started, so we can use it client_creator = protocol.ClientCreator(reactor, ssh.ClientTransport, self.username, self.conch_options, self.pub_key) create_defer = client_creator.connectTCP(self.hostname, self.config.port) # We're going to create a deferred, returned to the caller, that will # be called back when we have an established, secure connection ready # for opening channels. The value will be this instance of node. connect_defer = defer.Deferred() twistedutils.defer_timeout(connect_defer, self.node_settings.connect_timeout) def on_service_started(connection): # Booyah, time to start doing stuff if self.connection: log.error( "Host %s service started called before disconnect(%s, %s)", self.hostname, self.connection, connection) self.connection = connection self.connection_defer = None connect_defer.callback(self) return connection def on_connection_secure(connection): # We have a connection, but it might not be fully ready.... connection.service_start_defer = defer.Deferred() connection.service_stop_defer = defer.Deferred() connection.service_start_defer.addCallback(on_service_started) connection.service_stop_defer.addCallback(self._service_stopped) return connection def on_transport_create(transport): transport.connection_defer = defer.Deferred() transport.connection_defer.addCallback(on_connection_secure) return transport def on_transport_fail(fail): log.warning("Cannot connect to %s", self.hostname) connect_defer.errback(fail) create_defer.addCallback(on_transport_create) create_defer.addErrback(on_transport_fail) return connect_defer
def _connect(self): # This is complicated because we have to deal with a few different # steps before our connection is really available for us: # 1. Transport is created (our client creator does this) # 2. Our transport is secure, and we can create our connection # 3. The connection service is started, so we can use it client_creator = protocol.ClientCreator(reactor, ssh.ClientTransport, self.username, self.conch_options, self.pub_key) create_defer = client_creator.connectTCP(self.hostname, self.config.port) # We're going to create a deferred, returned to the caller, that will # be called back when we have an established, secure connection ready # for opening channels. The value will be this instance of node. connect_defer = defer.Deferred() twistedutils.defer_timeout(connect_defer, self.node_settings.connect_timeout) def on_service_started(connection): # Booyah, time to start doing stuff if self.connection: log.error("Host %s service started called before disconnect(%s, %s)", self.hostname, self.connection, connection) self.connection = connection self.connection_defer = None connect_defer.callback(self) return connection def on_connection_secure(connection): # We have a connection, but it might not be fully ready.... connection.service_start_defer = defer.Deferred() connection.service_stop_defer = defer.Deferred() connection.service_start_defer.addCallback(on_service_started) connection.service_stop_defer.addCallback(self._service_stopped) return connection def on_transport_create(transport): transport.connection_defer = defer.Deferred() transport.connection_defer.addCallback(on_connection_secure) return transport def on_transport_fail(fail): log.warning("Cannot connect to %s", self.hostname) connect_defer.errback(fail) create_defer.addCallback(on_transport_create) create_defer.addErrback(on_transport_fail) return connect_defer