def start(self): args = self.args fromdir = os.path.join(self.builder.basedir, self.args['fromdir']) todir = os.path.join(self.builder.basedir, self.args['todir']) self.timeout = args.get('timeout', 120) self.maxTime = args.get('maxTime', None) if runtime.platformType != "posix": d = threads.deferToThread(shutil.copytree, fromdir, todir) def cb(_): return 0 # rc=0 def eb(f): self.sendStatus( {'header': 'exception from copytree\n' + f.getTraceback()}) return -1 # rc=-1 d.addCallbacks(cb, eb) @d.addCallback def send_rc(rc): self.sendStatus({'rc': rc}) else: if not os.path.exists(os.path.dirname(todir)): os.makedirs(os.path.dirname(todir)) if os.path.exists(todir): # I don't think this happens, but just in case.. log.msg( "cp target '%s' already exists -- cp will not do what you think!" % todir) command = ['cp', '-R', '-P', '-p', '-v', fromdir, todir] c = runprocess.RunProcess(self.builder, command, self.builder.basedir, sendRC=False, timeout=self.timeout, maxTime=self.maxTime, logEnviron=self.logEnviron, usePTY=False) self.command = c d = c.start() d.addCallback(self._abandonOnFailure) d.addCallbacks(self._sendRC, self._checkAbandoned) return d
def testEnvironArray(self): b = FakeWorkerForBuilder(self.basedir) s = runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir, environ={"FOO": ['a', 'b']}) d = s.start() def check(ign): headers = "".join([ list(update.values())[0] for update in b.updates if list(update) == ["header"] ]) self.assertFalse( re.match('\bFOO=a{0}b\b'.format(os.pathsep), headers), "got:\n" + headers) d.addCallback(check) return d
def testEnvironPythonPath(self): b = FakeWorkerForBuilder(self.basedir) s = runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir, environ={"PYTHONPATH": 'a'}) d = s.start() def check(ign): headers = "".join([ list(update.values())[0] for update in b.updates if list(update) == ["header"] ]) self.assertFalse( re.match('\bPYTHONPATH=a%s' % (os.pathsep), headers), "got:\n" + headers) d.addCallback(check) return d
def test_with_child(self): # test that a process group gets killed parent_pidfile = self.new_pid_file() child_pidfile = self.new_pid_file() s = runprocess.RunProcess( scriptCommand('spawn_child', parent_pidfile, child_pidfile), self.basedir, 'utf-8', self.send_update) runproc_d = s.start() # wait for both processes to start up, then call s.kill parent_pid = yield self.wait_for_pidfile(parent_pidfile) child_pid = yield self.wait_for_pidfile(child_pidfile) s.kill("diaf") yield runproc_d self.assert_dead(parent_pid) self.assert_dead(child_pid)
def testMultiWordStringCommandQuotes(self): b = FakeWorkerForBuilder(self.basedir) # careful! This command must execute the same on windows and UNIX s = runprocess.RunProcess(b, 'echo "Happy Days and Jubilation"', self.basedir) if runtime.platformType == "win32": # echo doesn't parse out the quotes, so they come through in the # output exp = nl('"Happy Days and Jubilation"\n') else: exp = nl('Happy Days and Jubilation\n') d = s.start() def check(ign): self.assertTrue({'stdout': exp} in b.updates, b.show()) self.assertTrue({'rc': 0} in b.updates, b.show()) d.addCallback(check) return d
def test_sigterm(self, interruptSignal=None): # Tests that the process will receive SIGTERM if sigtermTimeout # is not None pidfile = self.newPidfile() self.pid = None b = FakeWorkerForBuilder(self.basedir) s = runprocess.RunProcess(b, scriptCommand( 'write_pidfile_and_sleep', pidfile), self.basedir, sigtermTime=1) runproc_d = s.start() pidfile_d = self.waitForPidfile(pidfile) self.receivedSIGTERM = False def check_alive(pid): # Create a mock process that will check if we receive SIGTERM mock_process = Mock(wraps=s.process) mock_process.pgid = None # Skips over group SIGTERM mock_process.pid = pid process = s.process def _mock_signalProcess(sig): if sig == "TERM": self.receivedSIGTERM = True process.signalProcess(sig) mock_process.signalProcess = _mock_signalProcess s.process = mock_process self.pid = pid # for use in check_dead # test that the process is still alive self.assertAlive(pid) # and tell the RunProcess object to kill it s.kill("diaf") pidfile_d.addCallback(check_alive) def check_dead(_): self.assertEqual(self.receivedSIGTERM, True) self.assertDead(self.pid) runproc_d.addCallback(check_dead) return defer.gatherResults([pidfile_d, runproc_d])
def do_test_pgroup(self, usePTY, useProcGroup=True, expectChildSurvival=False): # test that a process group gets killed parent_pidfile = self.newPidfile() self.parent_pid = None child_pidfile = self.newPidfile() self.child_pid = None b = FakeWorkerForBuilder(self.basedir) s = runprocess.RunProcess(b, scriptCommand( 'spawn_child', parent_pidfile, child_pidfile), self.basedir, usePTY=usePTY, useProcGroup=useProcGroup) runproc_d = s.start() # wait for both processes to start up, then call s.kill parent_pidfile_d = self.waitForPidfile(parent_pidfile) child_pidfile_d = self.waitForPidfile(child_pidfile) pidfiles_d = defer.gatherResults([parent_pidfile_d, child_pidfile_d]) def got_pids(pids): self.parent_pid, self.child_pid = pids pidfiles_d.addCallback(got_pids) def kill(_): s.kill("diaf") pidfiles_d.addCallback(kill) # check that both processes are dead after RunProcess is done d = defer.gatherResults([pidfiles_d, runproc_d]) def check_dead(_): self.assertDead(self.parent_pid) if expectChildSurvival: self.assertAlive(self.child_pid) else: self.assertDead(self.child_pid) d.addCallback(check_dead) return d
def testTrickyArguments(self): # make sure non-trivial arguments are passed verbatim b = FakeWorkerForBuilder(False, self.basedir) args = [ 'Happy Days and Jubilation', # spaces r'''!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~''', # special characters '%PATH%', # Windows variable expansions # Expansions get an argument of their own, because the Windows # shell doesn't treat % as special unless it surrounds a # variable name. ] s = runprocess.RunProcess(b, printArgsCommand() + args, self.basedir) d = s.start() def check(ign): self.failUnless({'stdout': nl(repr(args))} in b.updates, b.show()) self.failUnless({'rc': 0} in b.updates, b.show()) d.addCallback(check) return d
def test_simple(self, interrupt_signal=None): # test a simple process that just sleeps waiting to die pidfile = self.new_pid_file() b = FakeWorkerForBuilder(self.basedir) s = runprocess.RunProcess( b, scriptCommand('write_pidfile_and_sleep', pidfile), self.basedir) if interrupt_signal is not None: s.interruptSignal = interrupt_signal runproc_d = s.start() pid = yield self.wait_for_pidfile(pidfile) self.assert_alive(pid) # test that the process is still alive and tell the RunProcess object to kill it s.kill("diaf") yield runproc_d self.assert_dead(pid)
def do_test_double_fork(self, usePTY, useProcGroup=True, expectChildSurvival=False): # when a spawned process spawns another process, and then dies itself # (either intentionally or accidentally), we should be able to clean up # the child. parent_pidfile = self.newPidfile() self.parent_pid = None child_pidfile = self.newPidfile() self.child_pid = None b = FakeWorkerForBuilder(self.basedir) s = runprocess.RunProcess(b, scriptCommand( 'double_fork', parent_pidfile, child_pidfile), self.basedir, usePTY=usePTY, useProcGroup=useProcGroup) runproc_d = s.start() # wait for both processes to start up, then call s.kill parent_pidfile_d = self.waitForPidfile(parent_pidfile) child_pidfile_d = self.waitForPidfile(child_pidfile) pidfiles_d = defer.gatherResults([parent_pidfile_d, child_pidfile_d]) def got_pids(pids): self.parent_pid, self.child_pid = pids pidfiles_d.addCallback(got_pids) def kill(_): s.kill("diaf") pidfiles_d.addCallback(kill) # check that both processes are dead after RunProcess is done yield defer.gatherResults([pidfiles_d, runproc_d]) self.assertDead(self.parent_pid) if expectChildSurvival: self.assertAlive(self.child_pid) else: self.assertDead(self.child_pid)
def doClobber(self, dummy, dirname, chmodDone=False): d = os.path.join(self.builder.basedir, dirname) if runtime.platformType != "posix": d = threads.deferToThread(utils.rmdirRecursive, d) def cb(_): return 0 # rc=0 def eb(f): self.sendStatus({ 'header': 'exception from rmdirRecursive\n' + f.getTraceback() }) return -1 # rc=-1 d.addCallbacks(cb, eb) return d command = ["rm", "-rf", d] c = runprocess.RunProcess(self.builder, command, self.builder.basedir, sendRC=0, timeout=self.timeout, maxTime=self.maxTime, logEnviron=self.logEnviron, usePTY=False) self.command = c # sendRC=0 means the rm command will send stdout/stderr to the # master, but not the rc=0 when it finishes. That job is left to # _sendRC d = c.start() # The rm -rf may fail if there is a left-over subdir with chmod 000 # permissions. So if we get a failure, we attempt to chmod suitable # permissions and re-try the rm -rf. if chmodDone: d.addCallback(self._abandonOnFailure) else: d.addCallback(lambda rc: self.doClobberTryChmodIfFail(rc, dirname)) return d
def _clobber(self, dummy, chmodDone=False): command = ["rm", "-rf", self.dir] c = runprocess.RunProcess(self.builder, command, self.builder.basedir, sendRC=0, timeout=self.timeout, maxTime=self.maxTime, logEnviron=self.logEnviron, usePTY=False) self.command = c # sendRC=0 means the rm command will send stdout/stderr to the # master, but not the rc=0 when it finishes. That job is left to # _sendRC d = c.start() # The rm -rf may fail if there is a left-over subdir with chmod 000 # permissions. So if we get a failure, we attempt to chmod suitable # permissions and re-try the rm -rf. if not chmodDone: d.addCallback(self._tryChmod) return d
def test_incrementalDecoder(self): b = FakeWorkerForBuilder(self.basedir) b.unicode_encoding = "utf-8" s = runprocess.RunProcess(b, stderrCommand("hello"), self.basedir, sendStderr=True) pp = runprocess.RunProcessPP(s) # u"\N{SNOWMAN} when encoded to utf-8 bytes is b"\xe2\x98\x83" pp.outReceived(b"\xe2") pp.outReceived(b"\x98\x83") pp.errReceived(b"\xe2") pp.errReceived(b"\x98\x83") d = s.start() def check(ign): self.assertTrue({'stderr': u"\N{SNOWMAN}"} in b.updates) self.assertTrue({'stdout': u"\N{SNOWMAN}"} in b.updates) self.assertTrue({'rc': 0} in b.updates, b.show()) d.addCallback(check) return d
def doCopy(self, res): # now copy tree to workdir fromdir = os.path.join(self.builder.basedir, self.srcdir) todir = os.path.join(self.builder.basedir, self.workdir) if runtime.platformType != "posix": d = threads.deferToThread(shutil.copytree, fromdir, todir) def cb(_): return 0 # rc=0 def eb(f): self.sendStatus( {'header': 'exception from copytree\n' + f.getTraceback()}) return -1 # rc=-1 d.addCallbacks(cb, eb) return d if not os.path.exists(os.path.dirname(todir)): os.makedirs(os.path.dirname(todir)) if os.path.exists(todir): # I don't think this happens, but just in case.. log.msg( "cp target '%s' already exists -- cp will not do what you think!" % todir) command = ['cp', '-R', '-P', '-p', fromdir, todir] c = runprocess.RunProcess(self.builder, command, self.builder.basedir, sendRC=False, timeout=self.timeout, maxTime=self.maxTime, logEnviron=self.logEnviron, usePTY=False) self.command = c d = c.start() d.addCallback(self._abandonOnFailure) return d
def testEnvironExpandVar(self): b = FakeWorkerForBuilder(self.basedir) environ = { "EXPND": "-${PATH}-", "DOESNT_EXPAND": "-${---}-", "DOESNT_FIND": "-${DOESNT_EXISTS}-" } s = runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir, environ=environ) yield s.start() headers = "".join([ list(update.values())[0] for update in b.updates if list(update) == ["header"] ]) self.assertTrue("EXPND=-$" not in headers, "got:\n" + headers) self.assertTrue("DOESNT_FIND=--" in headers, "got:\n" + headers) self.assertTrue("DOESNT_EXPAND=-${---}-" in headers, "got:\n" + headers)
def _tryChmod(self, rc): assert isinstance(rc, int) if rc == 0: return defer.succeed(0) # Attempt a recursive chmod and re-try the rm -rf after. command = ["chmod", "-Rf", "u+rwx", os.path.join(self.builder.basedir, self.dir)] if sys.platform.startswith('freebsd'): # Work around a broken 'chmod -R' on FreeBSD (it tries to recurse into a # directory for which it doesn't have permission, before changing that # permission) by running 'find' instead command = ["find", os.path.join(self.builder.basedir, self.dir), '-exec', 'chmod', 'u+rwx', '{}', ';'] c = runprocess.RunProcess(self.builder, command, self.builder.basedir, sendRC=0, timeout=self.timeout, maxTime=self.maxTime, logEnviron=self.logEnviron, usePTY=False) self.command = c d = c.start() d.addCallback(lambda dummy: self._clobber(dummy, True)) return d
def test_with_child_parent_dies(self): # when a spawned process spawns another process, and then dies itself # (either intentionally or accidentally), we can't kill the child process. # In the future we should be able to fix this as Windows has CREATE_NEW_PROCESS_GROUP. parent_pidfile = self.new_pid_file() child_pidfile = self.new_pid_file() s = runprocess.RunProcess( scriptCommand('double_fork', parent_pidfile, child_pidfile), self.basedir, 'utf-8', self.send_update) runproc_d = s.start() # wait for both processes to start up, then call s.kill parent_pid = yield self.wait_for_pidfile(parent_pidfile) child_pid = yield self.wait_for_pidfile(child_pidfile) s.kill("diaf") # check that both processes are dead after RunProcess is done yield runproc_d self.assert_dead(parent_pid) self.assert_alive(child_pid)
def _test_spawnAsBatch(self, cmd, comspec): def spawnProcess(processProtocol, executable, args=(), env=None, path=None, uid=None, gid=None, usePTY=False, childFDs=None): self.assertTrue(args[0].lower().endswith("cmd.exe"), "{0} is not cmd.exe".format(args[0])) self.patch(runprocess.reactor, "spawnProcess", spawnProcess) tempEnviron = os.environ.copy() if 'COMSPEC' not in tempEnviron: tempEnviron['COMSPEC'] = comspec self.patch(os, "environ", tempEnviron) s = runprocess.RunProcess(cmd, self.basedir, 'utf-8', self.send_update) s.pp = runprocess.RunProcessPP(s) s.deferred = defer.Deferred() d = s._spawnAsBatch(s.pp, s.command, "args", tempEnviron, "path", False) return d
def _clobber(self, dummy, path, chmodDone=False): command = ["rm", "-rf", path] c = runprocess.RunProcess(command, self.protocol_command.worker_basedir, self.protocol_command.unicode_encoding, self.protocol_command.send_update, sendRC=0, timeout=self.timeout, maxTime=self.maxTime, logEnviron=self.logEnviron, usePTY=False) self.command = c # sendRC=0 means the rm command will send stdout/stderr to the # master, but not the rc=0 when it finishes. That job is left to # _sendRC rc = yield c.start() # The rm -rf may fail if there is a left-over subdir with chmod 000 # permissions. So if we get a failure, we attempt to chmod suitable # permissions and re-try the rm -rf. if not chmodDone: rc = yield self._tryChmod(rc, path) defer.returnValue(rc)
def test_startCommand_exception(self): b = FakeWorkerForBuilder(self.basedir) s = runprocess.RunProcess(b, ['whatever'], self.basedir) # set up to cause an exception in _startCommand def _startCommand(*args, **kwargs): raise RuntimeError() s._startCommand = _startCommand d = s.start() def check(err): err.trap(AbandonChain) stderr = [] # Here we're checking that the exception starting up the command # actually gets propagated back to the master in stderr. for u in b.updates: if 'stderr' in u: stderr.append(u['stderr']) stderr = "".join(stderr) self.assertTrue("RuntimeError" in stderr, stderr) d.addBoth(check) d.addBoth(lambda _: self.flushLoggedErrors()) return d
def testSendStatus(self): b = FakeWorkerForBuilder(self.basedir) s = runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir) s.sendStatus({'stdout': nl('hello\n')}) self.assertEqual(b.updates, [{'stdout': nl('hello\n')}], b.show())
def testObfuscatedCommand(self): b = FakeWorkerForBuilder(self.basedir) s = runprocess.RunProcess(b, [('obfuscated', 'abcd', 'ABCD')], self.basedir) self.assertEqual(s.command, [b'abcd']) self.assertEqual(s.fake_command, [b'ABCD'])
def testCommandEncodingObfuscated(self): b = FakeWorkerForBuilder(self.basedir) s = runprocess.RunProcess(b, [bsutil.Obfuscated(u'abcd', u'ABCD')], self.basedir) self.assertIsInstance(s.command[0], bytes) self.assertIsInstance(s.fake_command[0], bytes)
def testCommandEncodingList(self): b = FakeWorkerForBuilder(self.basedir) s = runprocess.RunProcess(b, [u'abcd', 'efg'], self.basedir) self.assertIsInstance(s.command[0], str) self.assertIsInstance(s.fake_command[0], str)
def makeRP(self): rp = runprocess.RunProcess(stdoutCommand('hello'), self.basedir, 'utf-8', self.send_update) return rp
def makeRP(self): b = FakeWorkerForBuilder(self.basedir) rp = runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir) return rp
def testSendNotimeout(self): b = FakeWorkerForBuilder(self.basedir) s = runprocess.RunProcess(b, stdoutCommand('hello'), self.basedir) data = "x" * (runprocess.RunProcess.BUFFER_SIZE + 1) s._addToBuffers('stdout', data) self.assertEqual(len(b.updates), 1)
def testCommandEncodingObfuscated(self): s = runprocess.RunProcess([bsutil.Obfuscated(u'abcd', u'ABCD')], self.basedir, 'utf-8', self.send_update) self.assertIsInstance(s.command[0], bytes) self.assertIsInstance(s.fake_command[0], bytes)
def testCommandEncoding(self): b = FakeWorkerForBuilder(self.basedir) s = runprocess.RunProcess(b, u'abcd', self.basedir) self.assertIsInstance(s.command, bytes) self.assertIsInstance(s.fake_command, bytes)
def testObfuscatedCommand(self): s = runprocess.RunProcess([('obfuscated', 'abcd', 'ABCD')], self.basedir, 'utf-8', self.send_update) self.assertEqual(s.command, [b'abcd']) self.assertEqual(s.fake_command, [b'ABCD'])