def setBuilddir(self, builddir):
assert self.parent
self.builddir = builddir
self.basedir = os.path.join(bytes2unicode(self.bot.basedir),
bytes2unicode(self.builddir))
if not os.path.isdir(self.basedir):
os.makedirs(self.basedir)
def _spawnAsBatch(self, processProtocol, executable, args, env, path,
usePTY):
"""A cheat that routes around the impedance mismatch between
twisted and cmd.exe with respect to escaping quotes"""
tf = NamedTemporaryFile(mode='w+',
dir='.',
suffix=".bat",
delete=False)
# echo off hides this cheat from the log files.
tf.write(u"@echo off\n")
if isinstance(self.command, (string_types, bytes)):
tf.write(bytes2unicode(self.command,
self.builder.unicode_encoding))
else:
tf.write(win32_batch_quote(self.command))
tf.close()
argv = os.environ['COMSPEC'].split() # allow %COMSPEC% to have args
if '/c' not in argv:
argv += ['/c']
argv += [tf.name]
def unlink_temp(result):
os.unlink(tf.name)
return result
self.deferred.addBoth(unlink_temp)
return reactor.spawnProcess(processProtocol,
executable,
argv,
env,
path,
usePTY=usePTY)
def escape_arg(arg):
arg = bytes2unicode(arg, unicode_encoding)
arg = quoteArguments([arg])
# escape shell special characters
arg = re.sub(r'[@()^"<>&|]', r'^\g<0>', arg)
# prevent variable expansion
return arg.replace('%', '%%')
def shell_quote(cmd_list, unicode_encoding='utf-8'):
# attempt to quote cmd_list such that a shell will properly re-interpret
# it. The pipes module is only available on UNIX; also, the quote
# function is undocumented (although it looks like it will be documented
# soon: http://bugs.python.org/issue9723). Finally, it has a nasty bug
# in some versions where an empty string is not quoted.
#
# So:
# - use pipes.quote on UNIX, handling '' as a special case
# - use our own custom function on Windows
if isinstance(cmd_list, bytes):
cmd_list = bytes2unicode(cmd_list, unicode_encoding)
if runtime.platformType == 'win32':
return win32_batch_quote(cmd_list, unicode_encoding)
# only available on unix
import pipes # pylint: disable=import-outside-toplevel
def quote(e):
if not e:
return u'""'
e = bytes2unicode(e, unicode_encoding)
return pipes.quote(e)
return u" ".join([quote(e) for e in cmd_list])
def escape_arg(arg):
arg = bytes2unicode(arg, unicode_encoding)
arg = quoteArguments([arg])
# escape shell special characters
arg = re.sub(r'[@()^"<>&|]', r'^\g<0>', arg)
# prevent variable expansion
return arg.replace('%', '%%')
def _spawnAsBatch(self, processProtocol, executable, args, env,
path, usePTY):
"""A cheat that routes around the impedance mismatch between
twisted and cmd.exe with respect to escaping quotes"""
tf = NamedTemporaryFile(mode='w+', dir='.', suffix=".bat",
delete=False)
# echo off hides this cheat from the log files.
tf.write(u"@echo off\n")
if isinstance(self.command, (string_types, bytes)):
tf.write(bytes2unicode(self.command, self.builder.unicode_encoding))
else:
tf.write(win32_batch_quote(self.command))
tf.close()
argv = os.environ['COMSPEC'].split() # allow %COMSPEC% to have args
if '/c' not in argv:
argv += ['/c']
argv += [tf.name]
def unlink_temp(result):
os.unlink(tf.name)
return result
self.deferred.addBoth(unlink_temp)
return reactor.spawnProcess(processProtocol, executable, argv, env,
path, usePTY=usePTY)
def decode(data, encoding='utf-8', errors='strict'):
"""We need to convert a dictionary where keys and values
are bytes, to unicode strings. This happens when a
Python 2 master sends a dictionary back to a Python 3 worker.
"""
data_type = type(data)
if data_type == bytes:
return bytes2unicode(data, encoding, errors)
if data_type in (dict, list, tuple):
if data_type == dict:
data = iteritems(data)
return data_type(map(decode, data))
return data
def decode(data, encoding='utf-8', errors='strict'):
"""We need to convert a dictionary where keys and values
are bytes, to unicode strings. This happens when a
Python 2 master sends a dictionary back to a Python 3 worker.
"""
data_type = type(data)
if data_type == bytes:
return bytes2unicode(data, encoding, errors)
if data_type in (dict, list, tuple):
if data_type == dict:
data = iteritems(data)
return data_type(map(decode, data))
return data
def _collapseMsg(self, msg):
"""
Take msg, which is a dictionary of lists of output chunks, and
concatenate all the chunks into a single string
"""
retval = {}
for logname in msg:
data = u""
for m in msg[logname]:
m = bytes2unicode(m, self.builder.unicode_encoding)
data += m
if isinstance(logname, tuple) and logname[0] == 'log':
retval['log'] = (logname[1], data)
else:
retval[logname] = data
return retval
def _collapseMsg(self, msg):
"""
Take msg, which is a dictionary of lists of output chunks, and
concatenate all the chunks into a single string
"""
retval = {}
for logname in msg:
data = u""
for m in msg[logname]:
m = bytes2unicode(m, self.builder.unicode_encoding)
data += m
if isinstance(logname, tuple) and logname[0] == 'log':
retval['log'] = (logname[1], data)
else:
retval[logname] = data
return retval
def shell_quote(cmd_list, unicode_encoding='utf-8'):
# attempt to quote cmd_list such that a shell will properly re-interpret
# it. The pipes module is only available on UNIX; also, the quote
# function is undocumented (although it looks like it will be documented
# soon: http://bugs.python.org/issue9723). Finally, it has a nasty bug
# in some versions where an empty string is not quoted.
#
# So:
# - use pipes.quote on UNIX, handling '' as a special case
# - use our own custom function on Windows
if isinstance(cmd_list, bytes):
cmd_list = bytes2unicode(cmd_list, unicode_encoding)
if runtime.platformType == 'win32':
return win32_batch_quote(cmd_list, unicode_encoding)
import pipes # only available on unix
def quote(e):
if not e:
return u'""'
e = bytes2unicode(e, unicode_encoding)
return pipes.quote(e)
return u" ".join([quote(e) for e in cmd_list])
def calculate_basedir(self, builddir):
return os.path.join(bytes2unicode(self.basedir),
bytes2unicode(builddir))
def quote(e):
if not e:
return u'""'
e = bytes2unicode(e, unicode_encoding)
return pipes.quote(e)
def errReceived(self, data):
if self.debug:
log.msg("RunProcessPP.errReceived")
data = bytes2unicode(data, self.command.builder.unicode_encoding)
self.command.addStderr(data)
def _startCommand(self):
# ensure workdir exists
if not os.path.isdir(self.workdir):
os.makedirs(self.workdir)
log.msg("RunProcess._startCommand")
self.pp = RunProcessPP(self)
self.using_comspec = False
self.command = unicode2bytes(self.command,
encoding=self.builder.unicode_encoding)
if isinstance(self.command, bytes):
if runtime.platformType == 'win32':
# allow %COMSPEC% to have args
argv = os.environ['COMSPEC'].split()
if '/c' not in argv:
argv += ['/c']
argv += [self.command]
self.using_comspec = True
else:
# for posix, use /bin/sh. for other non-posix, well, doesn't
# hurt to try
argv = [b'/bin/sh', b'-c', self.command]
display = self.fake_command
else:
# On windows, CreateProcess requires an absolute path to the executable.
# When we call spawnProcess below, we pass argv[0] as the executable.
# So, for .exe's that we have absolute paths to, we can call directly
# Otherwise, we should run under COMSPEC (usually cmd.exe) to
# handle path searching, etc.
if (runtime.platformType == 'win32'
and not (bytes2unicode(
self.command[0],
self.builder.unicode_encoding).lower().endswith(".exe")
and os.path.isabs(self.command[0]))):
# allow %COMSPEC% to have args
argv = os.environ['COMSPEC'].split()
if '/c' not in argv:
argv += ['/c']
argv += list(self.command)
self.using_comspec = True
else:
argv = self.command
# Attempt to format this for use by a shell, although the process
# isn't perfect
display = shell_quote(self.fake_command,
self.builder.unicode_encoding)
display = bytes2unicode(display, self.builder.unicode_encoding)
# $PWD usually indicates the current directory; spawnProcess may not
# update this value, though, so we set it explicitly here. This causes
# weird problems (bug #456) on msys, though..
if not self.environ.get('MACHTYPE', None) == 'i686-pc-msys':
self.environ['PWD'] = os.path.abspath(self.workdir)
# self.stdin is handled in RunProcessPP.connectionMade
log.msg(u" " + display)
self._addToBuffers(u'header', display + u"\n")
# then comes the secondary information
msg = u" in dir {0}".format(self.workdir)
if self.timeout:
if self.timeout == 1:
unit = u"sec"
else:
unit = u"secs"
msg += u" (timeout {0} {1})".format(self.timeout, unit)
if self.maxTime:
if self.maxTime == 1:
unit = u"sec"
else:
unit = u"secs"
msg += u" (maxTime {0} {1})".format(self.maxTime, unit)
log.msg(u" " + msg)
self._addToBuffers(u'header', msg + u"\n")
msg = " watching logfiles {0}".format(self.logfiles)
log.msg(" " + msg)
self._addToBuffers('header', msg + u"\n")
# then the obfuscated command array for resolving unambiguity
msg = u" argv: {0}".format(self.fake_command)
log.msg(u" " + msg)
self._addToBuffers('header', msg + u"\n")
# then the environment, since it sometimes causes problems
if self.logEnviron:
msg = u" environment:\n"
env_names = sorted(self.environ.keys())
for name in env_names:
msg += u" {0}={1}\n".format(
bytes2unicode(name,
encoding=self.builder.unicode_encoding),
bytes2unicode(self.environ[name],
encoding=self.builder.unicode_encoding))
log.msg(u" environment:\n{0}".format(pprint.pformat(self.environ)))
self._addToBuffers(u'header', msg)
if self.initialStdin:
msg = u" writing {0} bytes to stdin".format(len(self.initialStdin))
log.msg(u" " + msg)
self._addToBuffers(u'header', msg + u"\n")
msg = u" using PTY: {0}".format(bool(self.usePTY))
log.msg(u" " + msg)
self._addToBuffers(u'header', msg + u"\n")
# put data into stdin and close it, if necessary. This will be
# buffered until connectionMade is called
if self.initialStdin:
self.pp.setStdin(self.initialStdin)
self.startTime = util.now(self._reactor)
# start the process
self.process = self._spawnProcess(self.pp,
argv[0],
argv,
self.environ,
self.workdir,
usePTY=self.usePTY)
# set up timeouts
if self.timeout:
self.ioTimeoutTimer = self._reactor.callLater(
self.timeout, self.doTimeout)
if self.maxTime:
self.maxTimeoutTimer = self._reactor.callLater(
self.maxTime, self.doMaxTimeout)
for w in self.logFileWatchers:
w.start()
def _startCommand(self):
# ensure workdir exists
if not os.path.isdir(self.workdir):
os.makedirs(self.workdir)
log.msg("RunProcess._startCommand")
self.pp = RunProcessPP(self)
self.using_comspec = False
self.command = unicode2bytes(self.command, encoding=self.builder.unicode_encoding)
if isinstance(self.command, bytes):
if runtime.platformType == 'win32':
# allow %COMSPEC% to have args
argv = os.environ['COMSPEC'].split()
if '/c' not in argv:
argv += ['/c']
argv += [self.command]
self.using_comspec = True
else:
# for posix, use /bin/sh. for other non-posix, well, doesn't
# hurt to try
argv = [b'/bin/sh', b'-c', self.command]
display = self.fake_command
else:
# On windows, CreateProcess requires an absolute path to the executable.
# When we call spawnProcess below, we pass argv[0] as the executable.
# So, for .exe's that we have absolute paths to, we can call directly
# Otherwise, we should run under COMSPEC (usually cmd.exe) to
# handle path searching, etc.
if (runtime.platformType == 'win32' and
not (bytes2unicode(self.command[0],
self.builder.unicode_encoding).lower().endswith(".exe") and
os.path.isabs(self.command[0]))):
# allow %COMSPEC% to have args
argv = os.environ['COMSPEC'].split()
if '/c' not in argv:
argv += ['/c']
argv += list(self.command)
self.using_comspec = True
else:
argv = self.command
# Attempt to format this for use by a shell, although the process
# isn't perfect
display = shell_quote(self.fake_command, self.builder.unicode_encoding)
display = bytes2unicode(display, self.builder.unicode_encoding)
# $PWD usually indicates the current directory; spawnProcess may not
# update this value, though, so we set it explicitly here. This causes
# weird problems (bug #456) on msys, though..
if not self.environ.get('MACHTYPE', None) == 'i686-pc-msys':
self.environ['PWD'] = os.path.abspath(self.workdir)
# self.stdin is handled in RunProcessPP.connectionMade
log.msg(u" " + display)
self._addToBuffers(u'header', display + u"\n")
# then comes the secondary information
msg = u" in dir {0}".format(self.workdir)
if self.timeout:
if self.timeout == 1:
unit = u"sec"
else:
unit = u"secs"
msg += u" (timeout {0} {1})".format(self.timeout, unit)
if self.maxTime:
if self.maxTime == 1:
unit = u"sec"
else:
unit = u"secs"
msg += u" (maxTime {0} {1})".format(self.maxTime, unit)
log.msg(u" " + msg)
self._addToBuffers(u'header', msg + u"\n")
msg = " watching logfiles {0}".format(self.logfiles)
log.msg(" " + msg)
self._addToBuffers('header', msg + u"\n")
# then the obfuscated command array for resolving unambiguity
msg = u" argv: {0}".format(self.fake_command)
log.msg(u" " + msg)
self._addToBuffers('header', msg + u"\n")
# then the environment, since it sometimes causes problems
if self.logEnviron:
msg = u" environment:\n"
env_names = sorted(self.environ.keys())
for name in env_names:
msg += u" {0}={1}\n".format(bytes2unicode(name, encoding=self.builder.unicode_encoding),
bytes2unicode(self.environ[name], encoding=self.builder.unicode_encoding))
log.msg(u" environment:\n{0}".format(pprint.pformat(self.environ)))
self._addToBuffers(u'header', msg)
if self.initialStdin:
msg = u" writing {0} bytes to stdin".format(len(self.initialStdin))
log.msg(u" " + msg)
self._addToBuffers(u'header', msg + u"\n")
msg = u" using PTY: {0}".format(bool(self.usePTY))
log.msg(u" " + msg)
self._addToBuffers(u'header', msg + u"\n")
# put data into stdin and close it, if necessary. This will be
# buffered until connectionMade is called
if self.initialStdin:
self.pp.setStdin(self.initialStdin)
self.startTime = util.now(self._reactor)
# start the process
self.process = self._spawnProcess(
self.pp, argv[0], argv,
self.environ,
self.workdir,
usePTY=self.usePTY)
# set up timeouts
if self.timeout:
self.ioTimeoutTimer = self._reactor.callLater(
self.timeout, self.doTimeout)
if self.maxTime:
self.maxTimeoutTimer = self._reactor.callLater(
self.maxTime, self.doMaxTimeout)
for w in self.logFileWatchers:
w.start()
def quote(e):
if not e:
return u'""'
e = bytes2unicode(e, unicode_encoding)
return pipes.quote(e)
