def test_run_stdout_stderr_exit(self):
"""Check that we get the expected stdout, stderr and exit code
"""
stdoutText = "output out o text"
stderrText = "error err e text"
exitCode = 42
pythonScript = ("import sys;"
"sys.stdout.write('%s');"
"sys.stderr.write('%s');"
"exit(%i);" % (stdoutText, stderrText, exitCode))
command = "%s -c %r" % (PYTHON_COMMAND, pythonScript)
disconnectedDeferred = defer.Deferred()
protocol = ProcessProtocol()
protocol.processEnded = disconnectedDeferred.callback
resultDeferred = self.inductor.run(command, uid=UID)
resultDeferred.addErrback(self.fail)
@resultDeferred.addCallback
def checkResult(res):
r_stdoutText, r_stderrText, r_exitCode = res
self.assertEqual(r_stdoutText, stdoutText,
"stdout not as expected")
self.assertEqual(r_stderrText, stderrText,
"stderr not as expected")
self.assertEqual(r_exitCode, exitCode, "unexpected exit code")
return resultDeferred
def test_run_stdout_stderr_exit(self):
"""Check that we get the expected stdout, stderr and exit code
"""
stdoutText = "output out o text"
stderrText = "error err e text"
exitCode = 42
pythonScript = ("import sys;"
"sys.stdout.write('%s');"
"sys.stderr.write('%s');"
"exit(%i);"
% (stdoutText, stderrText, exitCode))
command = "%s -c %r" % (PYTHON_COMMAND, pythonScript)
disconnectedDeferred = defer.Deferred()
protocol = ProcessProtocol()
protocol.processEnded = disconnectedDeferred.callback
resultDeferred = self.inductor.run(command, uid=UID)
resultDeferred.addErrback(self.fail)
@resultDeferred.addCallback
def checkResult(res):
r_stdoutText, r_stderrText, r_exitCode = res
self.assertEqual(r_stdoutText, stdoutText, "stdout not as expected")
self.assertEqual(r_stderrText, stderrText, "stderr not as expected")
self.assertEqual(r_exitCode, exitCode, "unexpected exit code")
return resultDeferred
def start(self):
p = ProcessProtocol()
self.buff = ""
self.protocol = p
p.outReceived = self.got_out
p.processEnded = self.got_exit
reactor.spawnProcess(p, self.cmd[0], self.cmd)
def processExited(self, reason):
""" Logging, git process exit.
Called when the child process exits.
"""
self.log("process exited", "gateway - git process")
ProcessProtocol.processExited(self, reason)
def connectionMade(self):
ProcessProtocol.connectionMade(self)
LineReceiver.connectionMade(self)
self.transport.disconnecting = False #erm. Needs this to fix a bug in Twisted?
self.send_to_client("hello", [self.communicator.mod.name])
if not self.messages_not_acknowledged:
self.client_started_processing_at = time.time()
self.messages_not_acknowledged += 1
def childDataReceived(self, childFD, data):
"""
Handle data received on the specific pipe for the C{_ampProtocol}.
"""
if childFD == _WORKER_AMP_STDOUT:
self._ampProtocol.dataReceived(data)
else:
ProcessProtocol.childDataReceived(self, childFD, data)
def start(self):
p = ProcessProtocol()
self.buff = ""
self.protocol = p
p.outReceived = self.got_out
p.processEnded = self.got_exit
reactor.spawnProcess(p, self.cmd[0], self.cmd)
def childDataReceived(self, childFD, data):
"""
Handle data received on the specific pipe for the C{_ampProtocol}.
"""
if childFD == _WORKER_AMP_STDOUT:
self._ampProtocol.dataReceived(data)
else:
ProcessProtocol.childDataReceived(self, childFD, data)
def childDataReceived(self, childFD, data):
if childFD == 1:
self.out_received(data)
elif childFD == 2:
self._eliot_stderr.data_received(data)
elif childFD == 3:
self._eliot_stream.data_received(data)
else:
ProcessProtocol.childDataReceived(self, childFD, data)
def processEnded(self, reason):
"""See `ProcessProtocol.processEnded`.
If the process dies and we're waiting to SIGKILL it, we can stop
waiting.
"""
ProcessProtocol.processEnded(self, reason)
if self._sigkill_delayed_call is not None:
self._sigkill_delayed_call.cancel()
self._sigkill_delayed_call = None
def processEnded(self, reason):
"""See `ProcessProtocol.processEnded`.
If the process dies and we're waiting to SIGKILL it, we can stop
waiting.
"""
ProcessProtocol.processEnded(self, reason)
if self._sigkill_delayed_call is not None:
self._sigkill_delayed_call.cancel()
self._sigkill_delayed_call = None
def test_execute_exitcode_0(self):
"""Check that we get the expected reason failure object
when a process exits with exit code 0.
"""
processEndedDeferred = defer.Deferred()
protocol = ProcessProtocol()
protocol.processEnded = processEndedDeferred.callback
processDeferred = self.inductor.execute(protocol, SUCCEEDING_COMMAND,
uid=UID)
self.assertFailure(processEndedDeferred, ProcessDone)
return defer.DeferredList([processDeferred, processEndedDeferred],
fireOnOneErrback=True)
def processEnded(self, reason):
""" Logging, git process end.
Called when the child process exits and all file descriptors associated
with it have been closed.
"""
self.log("process ended", "gateway - git process")
ProcessProtocol.processEnded(self, reason)
# Signal to the other end that the process has ended through
# the underlying SSHSessionProcessProtocol
self.protocol.processEnded(reason)
self._end_record()
def makeDeferredWithProcessProtocol():
"""Returns a (`Deferred`, `ProcessProtocol`) tuple.
The Deferred's `callback()` will be called (with None) if the
`ProcessProtocol` is called back indicating that no error occurred.
Its `errback()` will be called with the `Failure` reason otherwise.
"""
done = Deferred()
protocol = ProcessProtocol()
# Call the errback if the "failure" object indicates a non-zero exit.
protocol.processEnded = lambda reason: (done.errback(reason) if (
reason and not reason.check(ProcessDone)) else done.callback(None))
return done, protocol
def test_execute_exitcode_0(self):
"""Check that we get the expected reason failure object
when a process exits with exit code 0.
"""
processEndedDeferred = defer.Deferred()
protocol = ProcessProtocol()
protocol.processEnded = processEndedDeferred.callback
processDeferred = self.inductor.execute(protocol,
SUCCEEDING_COMMAND,
uid=UID)
self.assertFailure(processEndedDeferred, ProcessDone)
return defer.DeferredList([processDeferred, processEndedDeferred],
fireOnOneErrback=True)
def test_launchWorkerProcesses(self):
"""
Given a C{spawnProcess} function, C{launchWorkerProcess} launches a
python process with an existing path as its argument.
"""
protocols = [ProcessProtocol() for i in range(4)]
arguments = []
environment = {}
def fakeSpawnProcess(processProtocol,
executable,
args=(),
env={},
path=None,
uid=None,
gid=None,
usePTY=0,
childFDs=None):
arguments.append(executable)
arguments.extend(args)
environment.update(env)
self.runner.launchWorkerProcesses(fakeSpawnProcess, protocols, ["foo"])
self.assertEqual(arguments[0], arguments[1])
self.assertTrue(os.path.exists(arguments[2]))
self.assertEqual("foo", arguments[3])
self.assertEqual(os.pathsep.join(sys.path),
environment["TRIAL_PYTHONPATH"])
def test_launchWorkerProcesses(self):
"""
Given a C{spawnProcess} function, C{launchWorkerProcess} launches a
python process with an existing path as its argument.
"""
protocols = [ProcessProtocol() for i in range(4)]
arguments = []
environment = {}
def fakeSpawnProcess(
processProtocol,
executable,
args=(),
env={},
path=None,
uid=None,
gid=None,
usePTY=0,
childFDs=None,
):
arguments.append(executable)
arguments.extend(args)
environment.update(env)
self.runner.launchWorkerProcesses(fakeSpawnProcess, protocols, ["foo"])
self.assertEqual(arguments[0], arguments[1])
self.assertTrue(os.path.exists(arguments[2]))
self.assertEqual("foo", arguments[3])
# The child process runs with PYTHONPATH set to exactly the parent's
# import search path so that the child has a good chance of finding
# the same source files the parent would have found.
self.assertEqual(os.pathsep.join(sys.path), environment["PYTHONPATH"])
def _respond_failure(self, failure, operation_id, reboot):
logging.info("Shutdown request failed.")
failure_report = '\n'.join([
failure.value.data,
"",
"Attempting to force {operation}. Please note that if this "
"succeeds, Landscape will have no way of knowing and will still "
"mark this activity as having failed. It is recommended you check "
"the state of the machine manually to determine whether "
"{operation} succeeded.".format(
operation="reboot" if reboot else "shutdown")
])
deferred = self._respond(FAILED, failure_report, operation_id)
# Add another callback spawning the poweroff or reboot command (which
# seem more reliable in aberrant situations like a post-trusty release
# upgrade where upstart has been replaced with systemd). If this
# succeeds, we won't have any opportunity to report it and if it fails
# we'll already have responded indicating we're attempting to force
# the operation so either way there's no sense capturing output
protocol = ProcessProtocol()
command, args = self._get_command_and_args(protocol, reboot, True)
deferred.addCallback(
lambda _: self._process_factory.spawnProcess(
protocol, command, args=args))
return deferred
def test_eofReceivedClosesStdin(self):
# 'eofReceived' closes standard input when called while a command is
# running.
protocol = ProcessProtocol()
self.session.execCommand(protocol, 'cat /etc/hostname')
self.session.eofReceived()
self.assertEqual([('closeStdin', )], self.session._transport.log)
def test_openShell(self):
"""
The L{ISession} adapter of the L{IConchUser} powerup implements
C{openShell} so as to associate the given L{IProcessProtocol} with a
transport.
"""
proto = ProcessProtocol()
session = ISession(IConchUser(self.store))
# XXX See Twisted ticket #3864
proto.session = session
proto.write = lambda bytes: None
# XXX See #2895.
session.getPty(None, (123, 456, 789, 1000), None)
session.openShell(proto)
self.assertNotIdentical(proto.transport, None)
def test_openShell(self):
"""
The L{ISession} adapter of the L{IConchUser} powerup implements
C{openShell} so as to associate the given L{IProcessProtocol} with a
transport.
"""
proto = ProcessProtocol()
session = ISession(IConchUser(self.store))
# XXX See Twisted ticket #3864
proto.session = session
proto.write = lambda bytes: None
# XXX See #2895.
session.getPty(None, (123, 456, 789, 1000), None)
session.openShell(proto)
self.assertNotIdentical(proto.transport, None)
def spawnProcess(self, reactor):
"""
Call C{reactor.spawnProcess} with some simple arguments. Do this here
so that code object referenced by the stack frame has a C{co_filename}
attribute set to this file so that L{TestCase.assertWarns} can be used.
"""
reactor.spawnProcess(
ProcessProtocol(), sys.executable, [sys.executable, "-c", ""],
usePTY=self.usePTY)
def test_authentication_failure(self):
"""Check that we get auth failure with wring credentials.
"""
self.inductor.setCredentials(WRONG_USER, WRONG_PASSWORD)
protocol = ProcessProtocol()
processDeferred = self.inductor.execute(protocol,
SUCCEEDING_COMMAND,
uid=WRONG_USER)
return self.assertFailure(processDeferred, TooManyAuthFailures)
def test_closedDisconnectsIfProcessCantBeTerminated(self):
# 'closed' still calls 'loseConnection' on the transport, even if the
# OS raises an error when we try to SIGHUP the process.
protocol = ProcessProtocol()
# MockTransport will raise an OSError on signalProcess if the executed
# command is 'raise-os-error'.
self.session.execCommand(protocol, 'raise-os-error')
self.session.closed()
self.assertEqual([('loseConnection', )], self.session._transport.log)
def test_execute_nonzero_exitcode(self):
"""Check that we get the expected failure when a process exits
with a nonzero exit code.
"""
processEndedDeferred = defer.Deferred()
protocol = ProcessProtocol()
protocol.processEnded = processEndedDeferred.callback
processDeferred = self.inductor.execute(protocol, FAILING_COMMAND,
uid=UID)
# Process should return a ProcessTerminated failure
# when it exits with a nonzero exit code
self.assertFailure(processEndedDeferred, ProcessTerminated)
@processEndedDeferred.addErrback
def checkExitCode(failure):
exitCode = failure.value.exitCode
self.assertNotEqual(exitCode, 0)
return defer.DeferredList([processDeferred, processEndedDeferred],
fireOnOneErrback=True)
def test_closedDisconnectsIfProcessAlreadyTerminated(self):
# 'closed' still calls 'loseConnection' on the transport, even if the
# process is already terminated
protocol = ProcessProtocol()
# MockTransport will raise a ProcessExitedAlready on signalProcess if
# the executed command is 'already-terminated'.
self.session.execCommand(protocol, 'already-terminated')
self.session.closed()
self.assertEqual([('loseConnection', )], self.session._transport.log)
def test_execCommandSpawnsProcess(self):
# ExecOnlySession.execCommand spawns the appropriate process.
protocol = ProcessProtocol()
command = 'cat /etc/hostname'
self.session.execCommand(protocol, command)
executable, arguments = self.session.getCommandToRun(command)
self.assertEqual([
(protocol, executable, arguments, None, None, None, None, 0, None)
], self.reactor.log)
def childDataReceived(self,childFD, data): ProcessProtocol.childDataReceived(self,childFD, data) if childFD == 4: self.buffer4.write(data) for m in self.buffer4.getMessages(): if isinstance(m,AnswerMessage): self.answerReceived(m) elif isinstance(m,EventMessage): self.eventReceived(m) elif isinstance(m,InfoMessage): self.infoReceived(m) elif isinstance(m,AskMessage): self.asking(m) elif isinstance(m,CommandMessage): self.command(m) else: print "____________________________________" print m print "____________________________________"
def open(self):
"""start process if not running"""
if not self.__osdcat:
self.__osdcat = ProcessProtocol()
reactor.spawnProcess(self.__osdcat, '/usr/bin/osd_cat', args=['osd_cat',
'--align=center', '--outline=5', '--lines=1', '--delay=2', '--offset=10',
'--font=-adobe-courier-bold-r-normal--*-640-*-*-*-*' \
], env={'DISPLAY': ':0'})
logDebug(self, 'p', 'OsdCat started process')
reactor.callLater(self.closeTimeout, self.close)
def childDataReceived(self, childFD, data):
ProcessProtocol.childDataReceived(self, childFD, data)
if childFD == 4:
self.buffer4.write(data)
for m in self.buffer4.getMessages():
if isinstance(m, AnswerMessage):
self.answerReceived(m)
elif isinstance(m, EventMessage):
self.eventReceived(m)
elif isinstance(m, InfoMessage):
self.infoReceived(m)
elif isinstance(m, AskMessage):
self.asking(m)
elif isinstance(m, CommandMessage):
self.command(m)
else:
print "____________________________________"
print m
print "____________________________________"
def pauseAndResume(reactor):
try:
protocol = ProcessProtocol()
transport = reactor.spawnProcess(protocol,
pyExe, [pyExe, b"-c", b""],
usePTY=self.usePTY)
transport.pauseProducing()
transport.resumeProducing()
finally:
reactor.stop()
def _spawnServer(self):
protocol = ProcessProtocol()
env = os.environ.get('VIRTUAL_ENV', '/usr')
executable = os.path.join(env, 'bin', 'twistd')
no_pid_argument = '--pidfile='
args = [executable, no_pid_argument, '-noy', TAC_FILE_PATH]
env = {'DEBUG_SERVER': 'yes'}
t = reactor.spawnProcess(protocol, executable, args, env=env)
self.addCleanup(os.kill, t.pid, signal.SIGKILL)
self.addCleanup(t.loseConnection)
return self._sleep(1) # it takes a while to start server
def test_closedTerminatesProcessAndDisconnects(self):
# ExecOnlySession provides a 'closed' method that is generally
# responsible for killing the child process and cleaning things up.
# From the outside, it just looks like a successful no-op. From the
# inside, it tells the process transport to end the connection between
# the SSH server and the child process.
protocol = ProcessProtocol()
self.session.execCommand(protocol, 'cat /etc/hostname')
self.session.closed()
self.assertEqual([('signalProcess', 'HUP'), ('loseConnection', )],
self.session._transport.log)
def test_execute_nonzero_exitcode(self):
"""Check that we get the expected failure when a process exits
with a nonzero exit code.
"""
processEndedDeferred = defer.Deferred()
protocol = ProcessProtocol()
protocol.processEnded = processEndedDeferred.callback
processDeferred = self.inductor.execute(protocol,
FAILING_COMMAND,
uid=UID)
# Process should return a ProcessTerminated failure
# when it exits with a nonzero exit code
self.assertFailure(processEndedDeferred, ProcessTerminated)
@processEndedDeferred.addErrback
def checkExitCode(failure):
exitCode = failure.value.exitCode
self.assertNotEqual(exitCode, 0)
return defer.DeferredList([processDeferred, processEndedDeferred],
fireOnOneErrback=True)
def test_environment(self):
# The environment for the executed process can be specified in the
# ExecOnlySession constructor.
session = ExecOnlySession(self.avatar,
self.reactor,
environment={'FOO': 'BAR'})
protocol = ProcessProtocol()
session.execCommand(protocol, 'yes')
self.assertEqual({'FOO': 'BAR'}, session.environment)
self.assertEqual([(protocol, 'yes', ['yes'], {
'FOO': 'BAR'
}, None, None, None, 0, None)], self.reactor.log)
def __init__(self, name, callback=None):
"""
@param name: name or description (for logging only)
@type name: str
@param callback: function to call when forked process finished
@type callback: func
"""
ProcessProtocol()
self.logger = logging.getLogger()
self.name = name
self.callback = callback
def newSubProcess(self):
here, there = socketpair(AF_UNIX, SOCK_STREAM)
owp = OneWorkerProtocol()
serverTransport = UNIXServer(here, owp, None, None, 4321, self.reactor)
owp.makeConnection(serverTransport)
argv = [sys.executable, __main__.__file__, b'w']
self.reactor.spawnProcess(
ProcessProtocol(), argv[0], args=argv, env=os.environ.copy(),
childFDs={STDIN: 'w', STDOUT: 'r', STDERR: 'r',
MAGIC_FILE_DESCRIPTOR: there.fileno()}
)
there.close()
serverTransport.startReading()
self.openSubprocessConnections.append(owp)
def _with_subscription_manager(reactor, k8s_context, f):
"""
Call ``f`` with a subscription manager client.
Do this by forwarding the API port to the subscription manager pod and
pointing a ``network_client`` at it.
"""
print("Tunnelling to subscription manager...")
subscription_manager_pod = yield _get_subscription_manager_pod(
reactor,
k8s_context,
)
# Use kubectl for the port-forward since txkube does not yet support that
# API.
forwarder = reactor.spawnProcess(
ProcessProtocol(),
b"kubectl",
[
b"kubectl",
b"--context",
k8s_context,
b"port-forward",
subscription_manager_pod,
b"9009:8000",
],
env=environ,
)
subscription_manager_client = network_client(
b"http://127.0.0.1:9009/",
Agent(reactor),
)
while True:
try:
yield subscription_manager_client.get("abcxyz")
except (ConnectionRefusedError, ResponseNeverReceived):
yield deferLater(reactor, 1.0, lambda: None)
except UnexpectedResponseCode:
break
try:
yield f(subscription_manager_client)
finally:
forwarder.signalProcess(b"TERM")
def _spawnServer(self):
# Format the following command:
# /path/to/twistd --pidfile= -noy /path/to/server.tac
path = os.environ.get('VIRTUAL_ENV', '/usr')
twistd = os.path.join(path, 'bin', 'twistd')
args = [twistd, '--pidfile=', '-noy', TAC_FILE_PATH]
# Use a special environment when running twistd that allow passing of
# couch url using environment variable, used by gitlab ci with docker
env = {
'DEBUG_SERVER': 'yes', # run Users API on port 2424 without TLS
'SOLEDAD_COUCH_URL': self.couch_url, # used by gitlab ci
}
protocol = ProcessProtocol()
proc = reactor.spawnProcess(protocol, twistd, args, env=env)
self.addCleanup(os.kill, proc.pid, signal.SIGKILL)
self.addCleanup(proc.loseConnection)
d = self._wait_for_server()
return d
def errReceived(self, reason):
self.logger.warn("%s: process failed: %s" % (self.name, reason))
self.transport.loseConnection()
ProcessProtocol.errReceived(self, reason)
def outReceived(self, data):
self.logger.debug("%s: process data received: %s" % (self.name, data))
ProcessProtocol.outReceived(self, data)
def errReceived(self, reason):
self.logger.warn("%s: process failed: %s" % (self.name, reason))
self.transport.loseConnection()
ProcessProtocol.errReceived(self, reason)
def outReceived(self, data):
self.logger.debug("%s: process data received: %s" % (self.name, data))
ProcessProtocol.outReceived(self, data)
def makeConnection(self,p): self.process = p self.processinfo = psutil.Process(p.pid) ProcessProtocol.makeConnection(self,p)
def childConnectionLost(self, childFD): ProcessProtocol.childConnectionLost(self, childFD) print "ON a PERDU :",childFD
def __init__(self):
ProcessProtocol.__init__(self)
self._responses = {}
def makeConnection(self,p):
print "process ",p," de ",self.name," ok"
self.process_test = p
self.processinfo = psutil.Process(p.pid)
ProcessProtocol.makeConnection(self,p)
def outReceived(self, data):
self.logger.debug("SSH Tunnel: process data received: %s" % (data))
ProcessProtocol.outReceived(self, data)
def connectionMade(self):
ProcessProtocol.connectionMade(self)
self.original_pid = self.transport.pid
def childDataReceived(self, fd, data):
if fd == 352:
self.line_receiver.dataReceived(data)
else:
ProcessProtocol.childDataReceived(self, fd, data)
def __init__(self, type="", mode=""):
self.type = type
self.mode = mode
ProcessProtocol.__init__(self)
