def start(self):
args = self.args
fromdir = os.path.join(self.builder.basedir, self.args['fromdir'])
todir = os.path.join(self.builder.basedir, self.args['todir'])
self.timeout = args.get('timeout', 120)
self.maxTime = args.get('maxTime', None)
if runtime.platformType != "posix":
d = threads.deferToThread(shutil.copytree, fromdir, todir)
def cb(_):
return 0 # rc=0
def eb(f):
self.sendStatus(
{'header': 'exception from copytree\n' + f.getTraceback()})
return -1 # rc=-1
d.addCallbacks(cb, eb)
@d.addCallback
def send_rc(rc):
self.sendStatus({'rc': rc})
else:
if not os.path.exists(os.path.dirname(todir)):
os.makedirs(os.path.dirname(todir))
if os.path.exists(todir):
# I don't think this happens, but just in case..
log.msg(
"cp target '%s' already exists -- cp will not do what you think!" % todir)
command = ['cp', '-R', '-P', '-p', '-v', fromdir, todir]
c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
sendRC=False, timeout=self.timeout, maxTime=self.maxTime,
logEnviron=self.logEnviron, usePTY=False)
self.command = c
d = c.start()
d.addCallback(self._abandonOnFailure)
d.addCallbacks(self._sendRC, self._checkAbandoned)
return d
def testEnvironArray(self):
b = FakeWorkerForBuilder(self.basedir)
s = runprocess.RunProcess(b,
stdoutCommand('hello'),
self.basedir,
environ={"FOO": ['a', 'b']})
d = s.start()
def check(ign):
headers = "".join([
list(update.values())[0] for update in b.updates
if list(update) == ["header"]
])
self.assertFalse(
re.match('\bFOO=a{0}b\b'.format(os.pathsep), headers),
"got:\n" + headers)
d.addCallback(check)
return d
def testEnvironPythonPath(self):
b = FakeWorkerForBuilder(self.basedir)
s = runprocess.RunProcess(b,
stdoutCommand('hello'),
self.basedir,
environ={"PYTHONPATH": 'a'})
d = s.start()
def check(ign):
headers = "".join([
list(update.values())[0] for update in b.updates
if list(update) == ["header"]
])
self.assertFalse(
re.match('\bPYTHONPATH=a%s' % (os.pathsep), headers),
"got:\n" + headers)
d.addCallback(check)
return d
def test_with_child(self):
# test that a process group gets killed
parent_pidfile = self.new_pid_file()
child_pidfile = self.new_pid_file()
s = runprocess.RunProcess(
scriptCommand('spawn_child', parent_pidfile, child_pidfile),
self.basedir, 'utf-8', self.send_update)
runproc_d = s.start()
# wait for both processes to start up, then call s.kill
parent_pid = yield self.wait_for_pidfile(parent_pidfile)
child_pid = yield self.wait_for_pidfile(child_pidfile)
s.kill("diaf")
yield runproc_d
self.assert_dead(parent_pid)
self.assert_dead(child_pid)
def testMultiWordStringCommandQuotes(self):
b = FakeWorkerForBuilder(self.basedir)
# careful! This command must execute the same on windows and UNIX
s = runprocess.RunProcess(b, 'echo "Happy Days and Jubilation"',
self.basedir)
if runtime.platformType == "win32":
# echo doesn't parse out the quotes, so they come through in the
# output
exp = nl('"Happy Days and Jubilation"\n')
else:
exp = nl('Happy Days and Jubilation\n')
d = s.start()
def check(ign):
self.assertTrue({'stdout': exp} in b.updates, b.show())
self.assertTrue({'rc': 0} in b.updates, b.show())
d.addCallback(check)
return d
def test_sigterm(self, interruptSignal=None):
# Tests that the process will receive SIGTERM if sigtermTimeout
# is not None
pidfile = self.newPidfile()
self.pid = None
b = FakeWorkerForBuilder(self.basedir)
s = runprocess.RunProcess(b,
scriptCommand(
'write_pidfile_and_sleep', pidfile),
self.basedir, sigtermTime=1)
runproc_d = s.start()
pidfile_d = self.waitForPidfile(pidfile)
self.receivedSIGTERM = False
def check_alive(pid):
# Create a mock process that will check if we receive SIGTERM
mock_process = Mock(wraps=s.process)
mock_process.pgid = None # Skips over group SIGTERM
mock_process.pid = pid
process = s.process
def _mock_signalProcess(sig):
if sig == "TERM":
self.receivedSIGTERM = True
process.signalProcess(sig)
mock_process.signalProcess = _mock_signalProcess
s.process = mock_process
self.pid = pid # for use in check_dead
# test that the process is still alive
self.assertAlive(pid)
# and tell the RunProcess object to kill it
s.kill("diaf")
pidfile_d.addCallback(check_alive)
def check_dead(_):
self.assertEqual(self.receivedSIGTERM, True)
self.assertDead(self.pid)
runproc_d.addCallback(check_dead)
return defer.gatherResults([pidfile_d, runproc_d])
def do_test_pgroup(self, usePTY, useProcGroup=True,
expectChildSurvival=False):
# test that a process group gets killed
parent_pidfile = self.newPidfile()
self.parent_pid = None
child_pidfile = self.newPidfile()
self.child_pid = None
b = FakeWorkerForBuilder(self.basedir)
s = runprocess.RunProcess(b,
scriptCommand(
'spawn_child', parent_pidfile, child_pidfile),
self.basedir,
usePTY=usePTY,
useProcGroup=useProcGroup)
runproc_d = s.start()
# wait for both processes to start up, then call s.kill
parent_pidfile_d = self.waitForPidfile(parent_pidfile)
child_pidfile_d = self.waitForPidfile(child_pidfile)
pidfiles_d = defer.gatherResults([parent_pidfile_d, child_pidfile_d])
def got_pids(pids):
self.parent_pid, self.child_pid = pids
pidfiles_d.addCallback(got_pids)
def kill(_):
s.kill("diaf")
pidfiles_d.addCallback(kill)
# check that both processes are dead after RunProcess is done
d = defer.gatherResults([pidfiles_d, runproc_d])
def check_dead(_):
self.assertDead(self.parent_pid)
if expectChildSurvival:
self.assertAlive(self.child_pid)
else:
self.assertDead(self.child_pid)
d.addCallback(check_dead)
return d
def testTrickyArguments(self):
# make sure non-trivial arguments are passed verbatim
b = FakeWorkerForBuilder(False, self.basedir)
args = [
'Happy Days and Jubilation', # spaces
r'''!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~''', # special characters
'%PATH%', # Windows variable expansions
# Expansions get an argument of their own, because the Windows
# shell doesn't treat % as special unless it surrounds a
# variable name.
]
s = runprocess.RunProcess(b, printArgsCommand() + args, self.basedir)
d = s.start()
def check(ign):
self.failUnless({'stdout': nl(repr(args))} in b.updates, b.show())
self.failUnless({'rc': 0} in b.updates, b.show())
d.addCallback(check)
return d
def test_simple(self, interrupt_signal=None):
# test a simple process that just sleeps waiting to die
pidfile = self.new_pid_file()
b = FakeWorkerForBuilder(self.basedir)
s = runprocess.RunProcess(
b, scriptCommand('write_pidfile_and_sleep', pidfile), self.basedir)
if interrupt_signal is not None:
s.interruptSignal = interrupt_signal
runproc_d = s.start()
pid = yield self.wait_for_pidfile(pidfile)
self.assert_alive(pid)
# test that the process is still alive and tell the RunProcess object to kill it
s.kill("diaf")
yield runproc_d
self.assert_dead(pid)
def do_test_double_fork(self, usePTY, useProcGroup=True,
expectChildSurvival=False):
# when a spawned process spawns another process, and then dies itself
# (either intentionally or accidentally), we should be able to clean up
# the child.
parent_pidfile = self.newPidfile()
self.parent_pid = None
child_pidfile = self.newPidfile()
self.child_pid = None
b = FakeWorkerForBuilder(self.basedir)
s = runprocess.RunProcess(b,
scriptCommand(
'double_fork', parent_pidfile, child_pidfile),
self.basedir,
usePTY=usePTY,
useProcGroup=useProcGroup)
runproc_d = s.start()
# wait for both processes to start up, then call s.kill
parent_pidfile_d = self.waitForPidfile(parent_pidfile)
child_pidfile_d = self.waitForPidfile(child_pidfile)
pidfiles_d = defer.gatherResults([parent_pidfile_d, child_pidfile_d])
def got_pids(pids):
self.parent_pid, self.child_pid = pids
pidfiles_d.addCallback(got_pids)
def kill(_):
s.kill("diaf")
pidfiles_d.addCallback(kill)
# check that both processes are dead after RunProcess is done
yield defer.gatherResults([pidfiles_d, runproc_d])
self.assertDead(self.parent_pid)
if expectChildSurvival:
self.assertAlive(self.child_pid)
else:
self.assertDead(self.child_pid)
def doClobber(self, dummy, dirname, chmodDone=False):
d = os.path.join(self.builder.basedir, dirname)
if runtime.platformType != "posix":
d = threads.deferToThread(utils.rmdirRecursive, d)
def cb(_):
return 0 # rc=0
def eb(f):
self.sendStatus({
'header':
'exception from rmdirRecursive\n' + f.getTraceback()
})
return -1 # rc=-1
d.addCallbacks(cb, eb)
return d
command = ["rm", "-rf", d]
c = runprocess.RunProcess(self.builder,
command,
self.builder.basedir,
sendRC=0,
timeout=self.timeout,
maxTime=self.maxTime,
logEnviron=self.logEnviron,
usePTY=False)
self.command = c
# sendRC=0 means the rm command will send stdout/stderr to the
# master, but not the rc=0 when it finishes. That job is left to
# _sendRC
d = c.start()
# The rm -rf may fail if there is a left-over subdir with chmod 000
# permissions. So if we get a failure, we attempt to chmod suitable
# permissions and re-try the rm -rf.
if chmodDone:
d.addCallback(self._abandonOnFailure)
else:
d.addCallback(lambda rc: self.doClobberTryChmodIfFail(rc, dirname))
return d
def _clobber(self, dummy, chmodDone=False):
command = ["rm", "-rf", self.dir]
c = runprocess.RunProcess(self.builder,
command,
self.builder.basedir,
sendRC=0,
timeout=self.timeout,
maxTime=self.maxTime,
logEnviron=self.logEnviron,
usePTY=False)
self.command = c
# sendRC=0 means the rm command will send stdout/stderr to the
# master, but not the rc=0 when it finishes. That job is left to
# _sendRC
d = c.start()
# The rm -rf may fail if there is a left-over subdir with chmod 000
# permissions. So if we get a failure, we attempt to chmod suitable
# permissions and re-try the rm -rf.
if not chmodDone:
d.addCallback(self._tryChmod)
return d
def test_incrementalDecoder(self):
b = FakeWorkerForBuilder(self.basedir)
b.unicode_encoding = "utf-8"
s = runprocess.RunProcess(b,
stderrCommand("hello"),
self.basedir,
sendStderr=True)
pp = runprocess.RunProcessPP(s)
# u"\N{SNOWMAN} when encoded to utf-8 bytes is b"\xe2\x98\x83"
pp.outReceived(b"\xe2")
pp.outReceived(b"\x98\x83")
pp.errReceived(b"\xe2")
pp.errReceived(b"\x98\x83")
d = s.start()
def check(ign):
self.assertTrue({'stderr': u"\N{SNOWMAN}"} in b.updates)
self.assertTrue({'stdout': u"\N{SNOWMAN}"} in b.updates)
self.assertTrue({'rc': 0} in b.updates, b.show())
d.addCallback(check)
return d
def doCopy(self, res):
# now copy tree to workdir
fromdir = os.path.join(self.builder.basedir, self.srcdir)
todir = os.path.join(self.builder.basedir, self.workdir)
if runtime.platformType != "posix":
d = threads.deferToThread(shutil.copytree, fromdir, todir)
def cb(_):
return 0 # rc=0
def eb(f):
self.sendStatus(
{'header': 'exception from copytree\n' + f.getTraceback()})
return -1 # rc=-1
d.addCallbacks(cb, eb)
return d
if not os.path.exists(os.path.dirname(todir)):
os.makedirs(os.path.dirname(todir))
if os.path.exists(todir):
# I don't think this happens, but just in case..
log.msg(
"cp target '%s' already exists -- cp will not do what you think!"
% todir)
command = ['cp', '-R', '-P', '-p', fromdir, todir]
c = runprocess.RunProcess(self.builder,
command,
self.builder.basedir,
sendRC=False,
timeout=self.timeout,
maxTime=self.maxTime,
logEnviron=self.logEnviron,
usePTY=False)
self.command = c
d = c.start()
d.addCallback(self._abandonOnFailure)
return d
def testEnvironExpandVar(self):
b = FakeWorkerForBuilder(self.basedir)
environ = {
"EXPND": "-${PATH}-",
"DOESNT_EXPAND": "-${---}-",
"DOESNT_FIND": "-${DOESNT_EXISTS}-"
}
s = runprocess.RunProcess(b,
stdoutCommand('hello'),
self.basedir,
environ=environ)
yield s.start()
headers = "".join([
list(update.values())[0] for update in b.updates
if list(update) == ["header"]
])
self.assertTrue("EXPND=-$" not in headers, "got:\n" + headers)
self.assertTrue("DOESNT_FIND=--" in headers, "got:\n" + headers)
self.assertTrue("DOESNT_EXPAND=-${---}-" in headers,
"got:\n" + headers)
def _tryChmod(self, rc):
assert isinstance(rc, int)
if rc == 0:
return defer.succeed(0)
# Attempt a recursive chmod and re-try the rm -rf after.
command = ["chmod", "-Rf", "u+rwx",
os.path.join(self.builder.basedir, self.dir)]
if sys.platform.startswith('freebsd'):
# Work around a broken 'chmod -R' on FreeBSD (it tries to recurse into a
# directory for which it doesn't have permission, before changing that
# permission) by running 'find' instead
command = ["find", os.path.join(self.builder.basedir, self.dir),
'-exec', 'chmod', 'u+rwx', '{}', ';']
c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
sendRC=0, timeout=self.timeout, maxTime=self.maxTime,
logEnviron=self.logEnviron, usePTY=False)
self.command = c
d = c.start()
d.addCallback(lambda dummy: self._clobber(dummy, True))
return d
def test_with_child_parent_dies(self):
# when a spawned process spawns another process, and then dies itself
# (either intentionally or accidentally), we can't kill the child process.
# In the future we should be able to fix this as Windows has CREATE_NEW_PROCESS_GROUP.
parent_pidfile = self.new_pid_file()
child_pidfile = self.new_pid_file()
s = runprocess.RunProcess(
scriptCommand('double_fork', parent_pidfile, child_pidfile),
self.basedir, 'utf-8', self.send_update)
runproc_d = s.start()
# wait for both processes to start up, then call s.kill
parent_pid = yield self.wait_for_pidfile(parent_pidfile)
child_pid = yield self.wait_for_pidfile(child_pidfile)
s.kill("diaf")
# check that both processes are dead after RunProcess is done
yield runproc_d
self.assert_dead(parent_pid)
self.assert_alive(child_pid)
def _test_spawnAsBatch(self, cmd, comspec):
def spawnProcess(processProtocol,
executable,
args=(),
env=None,
path=None,
uid=None,
gid=None,
usePTY=False,
childFDs=None):
self.assertTrue(args[0].lower().endswith("cmd.exe"),
"{0} is not cmd.exe".format(args[0]))
self.patch(runprocess.reactor, "spawnProcess", spawnProcess)
tempEnviron = os.environ.copy()
if 'COMSPEC' not in tempEnviron:
tempEnviron['COMSPEC'] = comspec
self.patch(os, "environ", tempEnviron)
s = runprocess.RunProcess(cmd, self.basedir, 'utf-8', self.send_update)
s.pp = runprocess.RunProcessPP(s)
s.deferred = defer.Deferred()
d = s._spawnAsBatch(s.pp, s.command, "args", tempEnviron, "path",
False)
return d
def _clobber(self, dummy, path, chmodDone=False):
command = ["rm", "-rf", path]
c = runprocess.RunProcess(command,
self.protocol_command.worker_basedir,
self.protocol_command.unicode_encoding,
self.protocol_command.send_update,
sendRC=0,
timeout=self.timeout,
maxTime=self.maxTime,
logEnviron=self.logEnviron,
usePTY=False)
self.command = c
# sendRC=0 means the rm command will send stdout/stderr to the
# master, but not the rc=0 when it finishes. That job is left to
# _sendRC
rc = yield c.start()
# The rm -rf may fail if there is a left-over subdir with chmod 000
# permissions. So if we get a failure, we attempt to chmod suitable
# permissions and re-try the rm -rf.
if not chmodDone:
rc = yield self._tryChmod(rc, path)
defer.returnValue(rc)
def test_startCommand_exception(self):
b = FakeWorkerForBuilder(self.basedir)
s = runprocess.RunProcess(b, ['whatever'], self.basedir)
# set up to cause an exception in _startCommand
def _startCommand(*args, **kwargs):
raise RuntimeError()
s._startCommand = _startCommand
d = s.start()
def check(err):
err.trap(AbandonChain)
stderr = []
# Here we're checking that the exception starting up the command
# actually gets propagated back to the master in stderr.
for u in b.updates:
if 'stderr' in u:
stderr.append(u['stderr'])
stderr = "".join(stderr)
self.assertTrue("RuntimeError" in stderr, stderr)
d.addBoth(check)
d.addBoth(lambda _: self.flushLoggedErrors())
return d
def testSendStatus(self):
b = FakeWorkerForBuilder(self.basedir)
s = runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir)
s.sendStatus({'stdout': nl('hello\n')})
self.assertEqual(b.updates, [{'stdout': nl('hello\n')}], b.show())
def testObfuscatedCommand(self):
b = FakeWorkerForBuilder(self.basedir)
s = runprocess.RunProcess(b, [('obfuscated', 'abcd', 'ABCD')],
self.basedir)
self.assertEqual(s.command, [b'abcd'])
self.assertEqual(s.fake_command, [b'ABCD'])
def testCommandEncodingObfuscated(self):
b = FakeWorkerForBuilder(self.basedir)
s = runprocess.RunProcess(b, [bsutil.Obfuscated(u'abcd', u'ABCD')],
self.basedir)
self.assertIsInstance(s.command[0], bytes)
self.assertIsInstance(s.fake_command[0], bytes)
def testCommandEncodingList(self):
b = FakeWorkerForBuilder(self.basedir)
s = runprocess.RunProcess(b, [u'abcd', 'efg'], self.basedir)
self.assertIsInstance(s.command[0], str)
self.assertIsInstance(s.fake_command[0], str)
def makeRP(self):
rp = runprocess.RunProcess(stdoutCommand('hello'), self.basedir,
'utf-8', self.send_update)
return rp
def makeRP(self):
b = FakeWorkerForBuilder(self.basedir)
rp = runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir)
return rp
def testSendNotimeout(self):
b = FakeWorkerForBuilder(self.basedir)
s = runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir)
data = "x" * (runprocess.RunProcess.BUFFER_SIZE + 1)
s._addToBuffers('stdout', data)
self.assertEqual(len(b.updates), 1)
def testCommandEncodingObfuscated(self):
s = runprocess.RunProcess([bsutil.Obfuscated(u'abcd', u'ABCD')],
self.basedir, 'utf-8', self.send_update)
self.assertIsInstance(s.command[0], bytes)
self.assertIsInstance(s.fake_command[0], bytes)
def testCommandEncoding(self):
b = FakeWorkerForBuilder(self.basedir)
s = runprocess.RunProcess(b, u'abcd', self.basedir)
self.assertIsInstance(s.command, bytes)
self.assertIsInstance(s.fake_command, bytes)
def testObfuscatedCommand(self):
s = runprocess.RunProcess([('obfuscated', 'abcd', 'ABCD')],
self.basedir, 'utf-8', self.send_update)
self.assertEqual(s.command, [b'abcd'])
self.assertEqual(s.fake_command, [b'ABCD'])
