def _open_channel(self, run):
assert self.connection
if not self._is_run_id_tracked(run):
log.warning("Run %s no longer tracked (_open_channel)", run.id)
return
assert self.run_states[run.id].state < RUN_STATE_RUNNING
self.run_states[run.id].state = RUN_STATE_STARTING
chan = ssh.ExecChannel(conn=self.connection)
chan.addOutputCallback(run.write_stdout)
chan.addErrorCallback(run.write_stderr)
chan.addEndCallback(run.done)
chan.command = run.command
chan.start_defer = defer.Deferred()
chan.start_defer.addCallback(self._run_started, run)
chan.start_defer.addErrback(self._run_start_error, run)
chan.exit_defer = defer.Deferred()
chan.exit_defer.addCallback(self._channel_complete, run)
chan.exit_defer.addErrback(self._channel_complete_unknown, run)
twistedutils.defer_timeout(chan.start_defer, RUN_START_TIMEOUT)
self.run_states[run.id].channel = chan
# TODO: I believe this needs to be checking the health of the connection
# before trying to open a new channel. If the connection is gone it
# needs to re-establish, or if the connection is not responding
# we shouldn't create this new channel
self.connection.openChannel(chan)
def _open_channel(self, run):
assert self.connection
if not self._is_run_id_tracked(run):
log.warning("Run %s no longer tracked (_open_channel)", run.id)
return
assert self.run_states[run.id].state < RUN_STATE_RUNNING
self.run_states[run.id].state = RUN_STATE_STARTING
chan = ssh.ExecChannel(conn=self.connection)
chan.addOutputCallback(run.write_stdout)
chan.addErrorCallback(run.write_stderr)
chan.addEndCallback(run.done)
chan.command = run.command
chan.start_defer = defer.Deferred()
chan.start_defer.addCallback(self._run_started, run)
chan.start_defer.addErrback(self._run_start_error, run)
chan.exit_defer = defer.Deferred()
chan.exit_defer.addCallback(self._channel_complete, run)
chan.exit_defer.addErrback(self._channel_complete_unknown, run)
twistedutils.defer_timeout(chan.start_defer, RUN_START_TIMEOUT)
self.run_states[run.id].channel = chan
# TODO: I believe this needs to be checking the health of the connection
# before trying to open a new channel. If the connection is gone it
# needs to re-establish, or if the connection is not responding
# we shouldn't create this new channel
self.connection.openChannel(chan)
def _service_stopped(self, connection):
"""Called when the SSH service has disconnected fully.
We should be in a state where we know there are no runs in progress
because all the SSH channels should have disconnected them.
"""
assert self.connection is connection
self.connection = None
log.info("Service to %s stopped", self.hostname)
for run_id, run in self.run_states.iteritems():
if run.state == RUN_STATE_CONNECTING:
# Now we can trigger a reconnect and re-start any waiting runs.
self._connect_then_run(run)
elif run.state == RUN_STATE_RUNNING:
self._fail_run(run, None)
elif run.state == RUN_STATE_STARTING:
if run.channel and run.channel.start_defer is not None:
# This means our run IS still waiting to start. There
# should be an outstanding timeout sitting on this guy as
# well. We'll just short circut it.
twistedutils.defer_timeout(run.channel.start_defer, 0)
else:
# Doesn't seem like this should ever happen.
log.warning("Run %r caught in starting state, but"
" start_defer is over.", run_id)
self._fail_run(run, None)
else:
# Service ended. The open channels should know how to handle
# this (and cleanup) themselves, so if there should not be any
# runs except those waiting to connect
raise Error("Run %s in state %s when service stopped",
run_id, run.state)
def _open_channel(self, run):
assert self.connection
assert self.run_states[run.id].state < RUN_STATE_RUNNING
self.run_states[run.id].state = RUN_STATE_STARTING
chan = ssh.ExecChannel(conn=self.connection)
chan.addOutputCallback(run.write_stdout)
chan.addErrorCallback(run.write_stderr)
chan.addEndCallback(run.done)
chan.command = run.command
chan.start_defer = defer.Deferred()
chan.start_defer.addCallback(self._run_started, run)
chan.start_defer.addErrback(self._run_start_error, run)
chan.exit_defer = defer.Deferred()
chan.exit_defer.addCallback(self._channel_complete, run)
chan.exit_defer.addErrback(self._channel_complete_unknown, run)
twistedutils.defer_timeout(chan.start_defer, RUN_START_TIMEOUT)
self.run_states[run.id].channel = chan
self.connection.openChannel(chan)
def _service_stopped(self, connection):
"""Called when the SSH service has disconnected fully.
We should be in a state where we know there are no runs in progress
because all the SSH channels should have disconnected them.
"""
assert self.connection is connection
self.connection = None
log.info("Service to %s stopped", self.hostname)
for run_id, run in self.run_states.iteritems():
if run.state == RUN_STATE_CONNECTING:
# Now we can trigger a reconnect and re-start any waiting runs.
self._connect_then_run(run)
elif run.state == RUN_STATE_RUNNING:
self._fail_run(run, None)
elif run.state == RUN_STATE_STARTING:
if run.channel and run.channel.start_defer is not None:
# This means our run IS still waiting to start. There
# should be an outstanding timeout sitting on this guy as
# well. We'll just short circut it.
twistedutils.defer_timeout(run.channel.start_defer, 0)
else:
# Doesn't seem like this should ever happen.
log.warning("Run %r caught in starting state, but"
" start_defer is over.", run_id)
self._fail_run(run, None)
else:
# Service ended. The open channels should know how to handle
# this (and cleanup) themselves, so if there should not be any
# runs except those waiting to connect
raise Error("Run %s in state %s when service stopped",
run_id, run.state)
def _connect(self):
# This is complicated because we have to deal with a few different
# steps before our connection is really available for us:
# 1. Transport is created (our client creator does this)
# 2. Our transport is secure, and we can create our connection
# 3. The connection service is started, so we can use it
client_creator = protocol.ClientCreator(reactor, ssh.ClientTransport,
self.username,
self.conch_options,
self.pub_key)
create_defer = client_creator.connectTCP(self.hostname,
self.config.port)
# We're going to create a deferred, returned to the caller, that will
# be called back when we have an established, secure connection ready
# for opening channels. The value will be this instance of node.
connect_defer = defer.Deferred()
twistedutils.defer_timeout(connect_defer,
self.node_settings.connect_timeout)
def on_service_started(connection):
# Booyah, time to start doing stuff
if self.connection:
log.error(
"Host %s service started called before disconnect(%s, %s)",
self.hostname, self.connection, connection)
self.connection = connection
self.connection_defer = None
connect_defer.callback(self)
return connection
def on_connection_secure(connection):
# We have a connection, but it might not be fully ready....
connection.service_start_defer = defer.Deferred()
connection.service_stop_defer = defer.Deferred()
connection.service_start_defer.addCallback(on_service_started)
connection.service_stop_defer.addCallback(self._service_stopped)
return connection
def on_transport_create(transport):
transport.connection_defer = defer.Deferred()
transport.connection_defer.addCallback(on_connection_secure)
return transport
def on_transport_fail(fail):
log.warning("Cannot connect to %s", self.hostname)
connect_defer.errback(fail)
create_defer.addCallback(on_transport_create)
create_defer.addErrback(on_transport_fail)
return connect_defer
def _connect(self):
# This is complicated because we have to deal with a few different
# steps before our connection is really available for us:
# 1. Transport is created (our client creator does this)
# 2. Our transport is secure, and we can create our connection
# 3. The connection service is started, so we can use it
client_creator = protocol.ClientCreator(reactor,
ssh.ClientTransport, self.username, self.conch_options, self.pub_key)
create_defer = client_creator.connectTCP(self.hostname, self.config.port)
# We're going to create a deferred, returned to the caller, that will
# be called back when we have an established, secure connection ready
# for opening channels. The value will be this instance of node.
connect_defer = defer.Deferred()
twistedutils.defer_timeout(connect_defer, self.node_settings.connect_timeout)
def on_service_started(connection):
# Booyah, time to start doing stuff
if self.connection:
log.error("Host %s service started called before disconnect(%s, %s)", self.hostname, self.connection, connection)
self.connection = connection
self.connection_defer = None
connect_defer.callback(self)
return connection
def on_connection_secure(connection):
# We have a connection, but it might not be fully ready....
connection.service_start_defer = defer.Deferred()
connection.service_stop_defer = defer.Deferred()
connection.service_start_defer.addCallback(on_service_started)
connection.service_stop_defer.addCallback(self._service_stopped)
return connection
def on_transport_create(transport):
transport.connection_defer = defer.Deferred()
transport.connection_defer.addCallback(on_connection_secure)
return transport
def on_transport_fail(fail):
log.warning("Cannot connect to %s", self.hostname)
connect_defer.errback(fail)
create_defer.addCallback(on_transport_create)
create_defer.addErrback(on_transport_fail)
return connect_defer
