def cbResponse(resp):
if resp.code == 200:
resp.deliverBody(BuildServerExtractor(d))
else:
proto.errReceived('\n ! Unable to contact build server.\n\n')
reason = failure.Failure(ProcessTerminated(exitCode=127))
return proto.processEnded(reason)
def test_restart_fails(self):
"""
If an error occurs before the error checking timeout the activity will
be failed. Data printed by the process prior to the failure is
included in the activity's result text.
"""
message = {"type": "shutdown", "reboot": True, "operation-id": 100}
self.plugin.perform_shutdown(message)
def restart_failed(message_id):
self.assertTrue(self.broker_service.exchanger.is_urgent())
messages = self.broker_service.message_store.get_pending_messages()
self.assertEqual(len(messages), 1)
message = messages[0]
self.assertEqual(message["type"], "operation-result")
self.assertEqual(message["api"], b"3.2")
self.assertEqual(message["operation-id"], 100)
self.assertEqual(message["timestamp"], 0)
self.assertEqual(message["status"], FAILED)
self.assertIn(u"Failure text is reported.", message["result-text"])
# Check that after failing, we attempt to force the shutdown by
# switching the binary called
[spawn1_args, spawn2_args] = self.process_factory.spawns
protocol = spawn2_args[0]
self.assertIsInstance(protocol, ProcessProtocol)
self.assertEqual(spawn2_args[1:3],
("/sbin/reboot", ["/sbin/reboot"]))
[arguments] = self.process_factory.spawns
protocol = arguments[0]
protocol.result.addCallback(restart_failed)
protocol.childDataReceived(0, b"Failure text is reported.")
protocol.processEnded(Failure(ProcessTerminated(exitCode=1)))
return protocol.result
def test_kill_unexpected_exit(self):
"""
The :py:class:`Deferred` returned by
:py:meth:`_HTTPBinProcess.kill` errbacks with the failure when
it is not :py:class:`ProcessTerminated`, or its signal does
not match the expected signal.
"""
for error in [
ProcessTerminated(1, signal=signal.SIGIO),
ConnectionDone("Bye")
]:
httpbin_process = parent._HTTPBinProcess(https=False)
httpbin_process.server_description(self.reactor)
spawned_process = self.reactor.spawnedProcesses[-1]
spawned_process.send_stdout(
shared._HTTPBinDescription(host="host",
port=1234).to_json_bytes() + b'\n')
termination_deferred = httpbin_process.kill()
spawned_process.end_process(Failure(error))
self.assertIs(
self.failureResultOf(termination_deferred).value, error)
def test_restart_fails(self):
"""
If an error occurs before the error checking timeout the activity will
be failed. Data printed by the process prior to the failure is
included in the activity's result text.
"""
message = {"type": "shutdown", "reboot": False, "operation-id": 100}
self.plugin.perform_shutdown(message)
def restart_failed(message_id):
self.assertTrue(self.broker_service.exchanger.is_urgent())
self.assertEqual(
self.broker_service.message_store.get_pending_messages(),
[{
"type": "operation-result",
"api": b"3.2",
"operation-id": 100,
"timestamp": 0,
"status": FAILED,
"result-text": u"Failure text is reported."
}])
[arguments] = self.process_factory.spawns
protocol = arguments[0]
protocol.result.addCallback(restart_failed)
protocol.childDataReceived(0, "Failure text is reported.")
protocol.processEnded(Failure(ProcessTerminated(exitCode=1)))
return protocol.result
def test_kill(self):
"""
Kill terminates the process as quickly as the platform allows,
and the termination failure is suppressed.
"""
httpbin_process = parent._HTTPBinProcess(https=False)
httpbin_process.server_description(self.reactor)
spawned_process = self.spawned_process()
spawned_process.send_stdout(
shared._HTTPBinDescription(host="host", port=1234).to_json_bytes()
+ b'\n')
termination_deferred = httpbin_process.kill()
self.assertEqual(
spawned_process.returned_process_transport_state.signals,
['KILL'],
)
spawned_process.end_process(
Failure(ProcessTerminated(1, signal=signal.SIGKILL)), )
self.successResultOf(termination_deferred)
def test_basebackup_retry(self):
"""
do_backup will re-try the basebackup command until it succeeds, and
will not fire until it succeeds.
"""
xpg = self.xpg_with_backup_dir(data_exists=True,
backup_exists=False,
tinkle_exists=False)
xpg.preflight()
results = []
d = xpg.do_backup()
d.addCallback(results.append)
self.assertEquals(len(self.reactor.spawnedProcesses), 1)
proc = self.reactor.spawnedProcesses.pop()
self.assertTrue(proc._executable.endswith("/pg_basebackup"))
self.assertEquals(results, [])
# Partial failure; let's make sure that doesn't get in there.
STDOUT = 1
os.write(proc._childFDs[STDOUT], 'BAD')
proc.proto.processExited(Failure(ProcessTerminated(status=1)))
self.assertEquals(results, [])
self.reactor.advance(2.0)
self.assertEquals(len(self.reactor.spawnedProcesses), 1)
proc = self.reactor.spawnedProcesses.pop()
os.write(proc._childFDs[STDOUT], 'GOOD')
proc.proto.processExited(Failure(ProcessDone(0)))
self.assertEquals(results, [None])
self.assertEquals(xpg.backup_zip_file.getContent(), 'GOOD')
def test__propagates_errors_from_command(self):
proto = JSONPerLineProtocol(callback=lambda obj: None)
proto.connectionMade()
reason = Failure(ProcessTerminated(1))
proto.processEnded(reason)
with ExpectedException(ProcessTerminated):
yield proto.done
def request_exit_signal(self, data):
"""
When the server sends the command's exit status, record it for later
delivery to the protocol.
@param data: The network-order four byte representation of the exit
signal of the command.
@type data: L{bytes}
"""
shortSignalName, data = getNS(data)
coreDumped, data = bool(ord(data[0:1])), data[1:]
errorMessage, data = getNS(data)
languageTag, data = getNS(data)
signalName = "SIG{}".format(nativeString(shortSignalName))
signalID = getattr(signal, signalName, -1)
self._log.info(
"Process exited with signal {shortSignalName!r};"
" core dumped: {coreDumped};"
" error message: {errorMessage};"
" language: {languageTag!r}",
shortSignalName=shortSignalName,
coreDumped=coreDumped,
errorMessage=errorMessage.decode("utf-8"),
languageTag=languageTag,
)
self._reason = ProcessTerminated(None, signalID, None)
def return_path_error():
proto.errReceived('\n ! Invalid path.')
proto.errReceived(
'\n ! Syntax is: [email protected]:<app>.git where <app> is your app\'s name.\n\n'
)
return proto.processEnded(
failure.Failure(ProcessTerminated(exitCode=1)))
def loseConnection(self):
if self.closed: return
self.closed = 1
self.proto.inConnectionLost()
self.proto.outConnectionLost()
self.proto.errConnectionLost()
self.proto.processEnded(failure.Failure(ProcessTerminated(0, None, None)))
def request_exit_status(self, data):
stat = struct.unpack('>L', data)[0]
if stat:
res = ProcessTerminated(exitCode=stat)
else:
res = ProcessDone(stat)
self._protocol.commandExited(Failure(res))
def closed(self):
if self._exitCode or self._signal:
reason = failure.Failure(
ProcessTerminated(self._exitCode, self._signal, self.status))
else:
reason = failure.Failure(ProcessDone(status=self.status))
processProtocol = self.processProtocol
del self.processProtocol
processProtocol.processEnded(reason)
def test_bad_arguments_exit(self):
"""If the subprocess exits with exit code 2, the ``Deferred`` returned
from ``zfs_command`` errbacks with ``BadArguments``.
"""
reactor = FakeProcessReactor()
result = zfs_command(reactor, [b"-H", b"lalala"])
process_protocol = reactor.processes[0].processProtocol
process_protocol.processEnded(Failure(ProcessTerminated(2)))
self.failureResultOf(result, BadArguments)
def test_error_exit(self):
"""If the subprocess exits with exit code 1, the ``Deferred`` returned
from ``zfs_command`` errbacks with ``CommandFailed``.
"""
reactor = FakeProcessReactor()
result = zfs_command(reactor, [b"-H", b"lalala"])
process_protocol = reactor.processes[0].processProtocol
process_protocol.processEnded(Failure(ProcessTerminated(1)))
self.failureResultOf(result, CommandFailed)
def loseConnection(self):
"""
Disconnect the protocol associated with this transport.
"""
if self.closed:
return
self.closed = 1
self.proto.inConnectionLost()
self.proto.outConnectionLost()
self.proto.errConnectionLost()
self.proto.processEnded(failure.Failure(ProcessTerminated(255, None, None)))
def request_exit_signal(self, data):
"""
When the server sends the shell's exit status, record it for later
delivery to the protocol.
@param data: The network-order four byte representation of the exit
signal of the shell.
@type data: L{bytes}
"""
(signal, ) = unpack('>L', data)
self._reason = ProcessTerminated(None, signal, None)
def sync_spawn_process(process_protocol,
argv=(),
reactor_process=None,
timeout=None):
"""Execute and capture a process'es standard output."""
if reactor_process is None: # noqa: no-cover
from twisted.internet import reactor # noqa: no-cover
reactor_process = reactor # noqa: no-cover
import threading
event = threading.Event()
output = [None, None]
def _cb(result):
if hasattr(result, 'decode'): # noqa: no-cover
result = result.decode('utf-8')
output[0] = result.rstrip("\r\n")
event.set()
def _cbe(result):
output[0] = result
event.set()
def _main(args):
d, output[1] = async_spawn_process(process_protocol, args,
reactor_process)
d.addCallback(_cb)
d.addErrback(_cbe)
reactor_process.callFromThread(_main, argv)
if event.wait(timeout) is not True:
if output[1] is not None:
# We timed-out; kill that program!
try:
output[1].signalProcess('TERM')
output[1].signalProcess('KILL')
except ProcessExitedAlready: # noqa: no-cover
pass # noqa: no-cover
output[1].loseConnection()
# Now raise an exception with the standard exit code for timeout. See timeout(1).
raise ProcessTerminated(exitCode=124)
else: # noqa: no-cover
LOGGER.error(
"A time-out occurred without a process handle present.")
if isinstance(output[0], Failure):
output[0].raiseException()
return output[0]
def request_exit_status(self, data):
"""
When the server sends the command's exit status, record it for later
delivery to the protocol.
@param data: The network-order four byte representation of the exit
status of the command.
@type data: L{bytes}
"""
(status, ) = unpack(">L", data)
if status != 0:
self._reason = ProcessTerminated(status, None, None)
def test_other_exit(self):
"""
If the subprocess exits with exit code other than 0, 1 or 2, the
``Deferred`` returned from ``zfs_command`` errbacks with
whatever error the process exited with.
"""
reactor = FakeProcessReactor()
result = zfs_command(reactor, [b"-H", b"lalala"])
process_protocol = reactor.processes[0].processProtocol
exception = ProcessTerminated(99)
process_protocol.processEnded(Failure(exception))
self.assertEqual(self.failureResultOf(result).value, exception)
def test_unregisters_killer_failure(self):
"""
When the process fails, the before-shutdown event is unregistered.
"""
reactor = ProcessCoreReactor()
d = run(reactor, ['command', 'and', 'args'])
[process] = reactor.processes
process.processProtocol.processEnded(Failure(ProcessTerminated(1)))
self.failureResultOf(d)
self.assertEqual(reactor._triggers['shutdown'].before, [])
def test_process_dies_shortly_after_fork(self):
"""
If the service process exists right after having been spawned (for
example the executable was not found), the 'ready' Deferred fires
with an errback.
"""
self.protocol.makeConnection(self.process)
error = ProcessTerminated(exitCode=1, signal=None)
self.protocol.processExited(Failure(error))
self.assertThat(self.protocol.ready,
failed(MatchesStructure(value=Is(error))))
def test_processEndedWithExitSignalNoCoreDump(self):
"""
When processEnded is called, if there is an exit signal in the
reason it should be sent in an exit-signal message. If no
core was dumped, don't set the core-dump bit.
"""
self.pp.processEnded(Failure(ProcessTerminated(1, signal.SIGTERM, 0)))
# see comments in test_processEndedWithExitSignalCoreDump for the
# meaning of the parts in the request
self.assertRequestsEqual([
('exit-signal', NS('TERM') + '\x00' + NS('') + NS(''), False)
])
self.assertSessionClosed()
def test_process_failure(self):
"""
If the process ends with a failure, the returned deferred fires with
the reason.
"""
reactor = ProcessCoreReactor()
d = run(reactor, ['command', 'and', 'args'])
[process] = reactor.processes
expected_failure = Failure(ProcessTerminated(1))
process.processProtocol.processEnded(expected_failure)
self.assertEqual(self.failureResultOf(d), expected_failure)
def test_processEndedWithExitSignalCoreDump(self):
"""
When processEnded is called, if there is an exit signal in the reason
it should be sent in an exit-signal message. The connection should be
closed.
"""
self.pp.processEnded(
Failure(ProcessTerminated(1, signal.SIGTERM,
1 << 7))) # 7th bit means core dumped
self.assertRequestsEqual([(
'exit-signal',
common.NS('TERM') # signal name
+ '\x01' # core dumped is true
+ common.NS('') # error message
+ common.NS(''), # language tag
False)])
self.assertSessionClosed()
def async_spawn_process(process_protocol, args, reactor_process=None):
"""Execute a process using ```Twisted```."""
if reactor_process is None: # noqa: no-cover
from twisted.internet import reactor # noqa: no-cover
reactor_process = reactor # noqa: no-cover
try:
process = reactor_process.spawnProcess(process_protocol,
args[0],
args,
env=None)
return process_protocol.d, process
except OSError as e: # noqa: no-cover
if e.errno is None or e.errno == errno.ENOENT:
return defer.fail(ProcessTerminated(exitCode=1)), None
else:
return defer.fail(e), None
def test_process_dies_while_probing_port(self):
"""
If the service process exists while waiting for the expected port to,
be open, the 'ready' Deferred fires with an errback.
"""
self.protocol.expectedPort = 1234
self.protocol.makeConnection(self.process)
self.reactor.advance(self.protocol.minUptime)
error = ProcessTerminated(exitCode=1, signal=None)
self.protocol.processExited(Failure(error))
self.assertThat(self.protocol.ready,
failed(MatchesStructure(value=Is(error))))
# No further probe will happen
self.reactor.advance(0.1)
self.assertEqual(1, len(self.reactor.tcpClients))
def test_process_dies_while_waiting_expected_output(self):
"""
If the service process exists while waiting for the expected output,
the 'ready' Deferred fires with an errback.
"""
self.protocol.expectedOutput = "hello"
self.protocol.makeConnection(self.process)
self.reactor.advance(self.protocol.minUptime)
error = ProcessTerminated(exitCode=1, signal=None)
self.protocol.processExited(Failure(error))
self.assertThat(self.protocol.ready,
failed(MatchesStructure(value=Is(error))))
# Further input received on the file descriptor will be discarded
self.protocol.ready = Deferred() # pretend that we didn't get fired
self.protocol.outReceived(b"hello world!\n")
self.assertThat(self.protocol.ready, has_no_result())
def test_process_ends_after_timeout(self):
"""
If the process ends after the error checking timeout has passed
C{result} will not be re-fired.
"""
message = {"type": "shutdown", "reboot": False, "operation-id": 100}
self.plugin.perform_shutdown(message)
stash = []
def restart_performed(ignore):
self.assertEqual(stash, [])
stash.append(True)
[arguments] = self.process_factory.spawns
protocol = arguments[0]
protocol.result.addCallback(restart_performed)
self.manager.reactor.advance(10)
protocol.processEnded(Failure(ProcessTerminated(exitCode=1)))
return protocol.result
def __init__(self, proto, code, message):
proto.makeConnection(self)
proto.childDataReceived(2, message + '\n')
proto.childConnectionLost(0)
proto.childConnectionLost(1)
proto.childConnectionLost(2)
failure = Failure(ProcessTerminated(code))
proto.processExited(failure)
proto.processEnded(failure)
# ignore all unused methods
noop = lambda *args, **kwargs: None
self.closeStdin = noop
self.closeStdout = noop
self.closeStderr = noop
self.writeToChild = noop
self.loseConnection = noop
self.signalProcess = noop
def cbErrResponse(err):
proto.errReceived('\n ! Unable to contact build server.\n\n')
reason = failure.Failure(ProcessTerminated(exitCode=127))
return proto.processEnded(reason)