def setBuilddir(self, builddir): assert self.parent self.builddir = builddir self.basedir = os.path.join(bytes2unicode(self.bot.basedir), bytes2unicode(self.builddir)) if not os.path.isdir(self.basedir): os.makedirs(self.basedir)
def _spawnAsBatch(self, processProtocol, executable, args, env, path, usePTY): """A cheat that routes around the impedance mismatch between twisted and cmd.exe with respect to escaping quotes""" tf = NamedTemporaryFile(mode='w+', dir='.', suffix=".bat", delete=False) # echo off hides this cheat from the log files. tf.write(u"@echo off\n") if isinstance(self.command, (string_types, bytes)): tf.write(bytes2unicode(self.command, self.builder.unicode_encoding)) else: tf.write(win32_batch_quote(self.command)) tf.close() argv = os.environ['COMSPEC'].split() # allow %COMSPEC% to have args if '/c' not in argv: argv += ['/c'] argv += [tf.name] def unlink_temp(result): os.unlink(tf.name) return result self.deferred.addBoth(unlink_temp) return reactor.spawnProcess(processProtocol, executable, argv, env, path, usePTY=usePTY)
def escape_arg(arg): arg = bytes2unicode(arg, unicode_encoding) arg = quoteArguments([arg]) # escape shell special characters arg = re.sub(r'[@()^"<>&|]', r'^\g<0>', arg) # prevent variable expansion return arg.replace('%', '%%')
def shell_quote(cmd_list, unicode_encoding='utf-8'): # attempt to quote cmd_list such that a shell will properly re-interpret # it. The pipes module is only available on UNIX; also, the quote # function is undocumented (although it looks like it will be documented # soon: http://bugs.python.org/issue9723). Finally, it has a nasty bug # in some versions where an empty string is not quoted. # # So: # - use pipes.quote on UNIX, handling '' as a special case # - use our own custom function on Windows if isinstance(cmd_list, bytes): cmd_list = bytes2unicode(cmd_list, unicode_encoding) if runtime.platformType == 'win32': return win32_batch_quote(cmd_list, unicode_encoding) # only available on unix import pipes # pylint: disable=import-outside-toplevel def quote(e): if not e: return u'""' e = bytes2unicode(e, unicode_encoding) return pipes.quote(e) return u" ".join([quote(e) for e in cmd_list])
def escape_arg(arg): arg = bytes2unicode(arg, unicode_encoding) arg = quoteArguments([arg]) # escape shell special characters arg = re.sub(r'[@()^"<>&|]', r'^\g<0>', arg) # prevent variable expansion return arg.replace('%', '%%')
def _spawnAsBatch(self, processProtocol, executable, args, env, path, usePTY): """A cheat that routes around the impedance mismatch between twisted and cmd.exe with respect to escaping quotes""" tf = NamedTemporaryFile(mode='w+', dir='.', suffix=".bat", delete=False) # echo off hides this cheat from the log files. tf.write(u"@echo off\n") if isinstance(self.command, (string_types, bytes)): tf.write(bytes2unicode(self.command, self.builder.unicode_encoding)) else: tf.write(win32_batch_quote(self.command)) tf.close() argv = os.environ['COMSPEC'].split() # allow %COMSPEC% to have args if '/c' not in argv: argv += ['/c'] argv += [tf.name] def unlink_temp(result): os.unlink(tf.name) return result self.deferred.addBoth(unlink_temp) return reactor.spawnProcess(processProtocol, executable, argv, env, path, usePTY=usePTY)
def decode(data, encoding='utf-8', errors='strict'): """We need to convert a dictionary where keys and values are bytes, to unicode strings. This happens when a Python 2 master sends a dictionary back to a Python 3 worker. """ data_type = type(data) if data_type == bytes: return bytes2unicode(data, encoding, errors) if data_type in (dict, list, tuple): if data_type == dict: data = iteritems(data) return data_type(map(decode, data)) return data
def decode(data, encoding='utf-8', errors='strict'): """We need to convert a dictionary where keys and values are bytes, to unicode strings. This happens when a Python 2 master sends a dictionary back to a Python 3 worker. """ data_type = type(data) if data_type == bytes: return bytes2unicode(data, encoding, errors) if data_type in (dict, list, tuple): if data_type == dict: data = iteritems(data) return data_type(map(decode, data)) return data
def _collapseMsg(self, msg): """ Take msg, which is a dictionary of lists of output chunks, and concatenate all the chunks into a single string """ retval = {} for logname in msg: data = u"" for m in msg[logname]: m = bytes2unicode(m, self.builder.unicode_encoding) data += m if isinstance(logname, tuple) and logname[0] == 'log': retval['log'] = (logname[1], data) else: retval[logname] = data return retval
def _collapseMsg(self, msg): """ Take msg, which is a dictionary of lists of output chunks, and concatenate all the chunks into a single string """ retval = {} for logname in msg: data = u"" for m in msg[logname]: m = bytes2unicode(m, self.builder.unicode_encoding) data += m if isinstance(logname, tuple) and logname[0] == 'log': retval['log'] = (logname[1], data) else: retval[logname] = data return retval
def shell_quote(cmd_list, unicode_encoding='utf-8'): # attempt to quote cmd_list such that a shell will properly re-interpret # it. The pipes module is only available on UNIX; also, the quote # function is undocumented (although it looks like it will be documented # soon: http://bugs.python.org/issue9723). Finally, it has a nasty bug # in some versions where an empty string is not quoted. # # So: # - use pipes.quote on UNIX, handling '' as a special case # - use our own custom function on Windows if isinstance(cmd_list, bytes): cmd_list = bytes2unicode(cmd_list, unicode_encoding) if runtime.platformType == 'win32': return win32_batch_quote(cmd_list, unicode_encoding) import pipes # only available on unix def quote(e): if not e: return u'""' e = bytes2unicode(e, unicode_encoding) return pipes.quote(e) return u" ".join([quote(e) for e in cmd_list])
def calculate_basedir(self, builddir): return os.path.join(bytes2unicode(self.basedir), bytes2unicode(builddir))
def quote(e): if not e: return u'""' e = bytes2unicode(e, unicode_encoding) return pipes.quote(e)
def errReceived(self, data): if self.debug: log.msg("RunProcessPP.errReceived") data = bytes2unicode(data, self.command.builder.unicode_encoding) self.command.addStderr(data)
def _startCommand(self): # ensure workdir exists if not os.path.isdir(self.workdir): os.makedirs(self.workdir) log.msg("RunProcess._startCommand") self.pp = RunProcessPP(self) self.using_comspec = False self.command = unicode2bytes(self.command, encoding=self.builder.unicode_encoding) if isinstance(self.command, bytes): if runtime.platformType == 'win32': # allow %COMSPEC% to have args argv = os.environ['COMSPEC'].split() if '/c' not in argv: argv += ['/c'] argv += [self.command] self.using_comspec = True else: # for posix, use /bin/sh. for other non-posix, well, doesn't # hurt to try argv = [b'/bin/sh', b'-c', self.command] display = self.fake_command else: # On windows, CreateProcess requires an absolute path to the executable. # When we call spawnProcess below, we pass argv[0] as the executable. # So, for .exe's that we have absolute paths to, we can call directly # Otherwise, we should run under COMSPEC (usually cmd.exe) to # handle path searching, etc. if (runtime.platformType == 'win32' and not (bytes2unicode( self.command[0], self.builder.unicode_encoding).lower().endswith(".exe") and os.path.isabs(self.command[0]))): # allow %COMSPEC% to have args argv = os.environ['COMSPEC'].split() if '/c' not in argv: argv += ['/c'] argv += list(self.command) self.using_comspec = True else: argv = self.command # Attempt to format this for use by a shell, although the process # isn't perfect display = shell_quote(self.fake_command, self.builder.unicode_encoding) display = bytes2unicode(display, self.builder.unicode_encoding) # $PWD usually indicates the current directory; spawnProcess may not # update this value, though, so we set it explicitly here. This causes # weird problems (bug #456) on msys, though.. if not self.environ.get('MACHTYPE', None) == 'i686-pc-msys': self.environ['PWD'] = os.path.abspath(self.workdir) # self.stdin is handled in RunProcessPP.connectionMade log.msg(u" " + display) self._addToBuffers(u'header', display + u"\n") # then comes the secondary information msg = u" in dir {0}".format(self.workdir) if self.timeout: if self.timeout == 1: unit = u"sec" else: unit = u"secs" msg += u" (timeout {0} {1})".format(self.timeout, unit) if self.maxTime: if self.maxTime == 1: unit = u"sec" else: unit = u"secs" msg += u" (maxTime {0} {1})".format(self.maxTime, unit) log.msg(u" " + msg) self._addToBuffers(u'header', msg + u"\n") msg = " watching logfiles {0}".format(self.logfiles) log.msg(" " + msg) self._addToBuffers('header', msg + u"\n") # then the obfuscated command array for resolving unambiguity msg = u" argv: {0}".format(self.fake_command) log.msg(u" " + msg) self._addToBuffers('header', msg + u"\n") # then the environment, since it sometimes causes problems if self.logEnviron: msg = u" environment:\n" env_names = sorted(self.environ.keys()) for name in env_names: msg += u" {0}={1}\n".format( bytes2unicode(name, encoding=self.builder.unicode_encoding), bytes2unicode(self.environ[name], encoding=self.builder.unicode_encoding)) log.msg(u" environment:\n{0}".format(pprint.pformat(self.environ))) self._addToBuffers(u'header', msg) if self.initialStdin: msg = u" writing {0} bytes to stdin".format(len(self.initialStdin)) log.msg(u" " + msg) self._addToBuffers(u'header', msg + u"\n") msg = u" using PTY: {0}".format(bool(self.usePTY)) log.msg(u" " + msg) self._addToBuffers(u'header', msg + u"\n") # put data into stdin and close it, if necessary. This will be # buffered until connectionMade is called if self.initialStdin: self.pp.setStdin(self.initialStdin) self.startTime = util.now(self._reactor) # start the process self.process = self._spawnProcess(self.pp, argv[0], argv, self.environ, self.workdir, usePTY=self.usePTY) # set up timeouts if self.timeout: self.ioTimeoutTimer = self._reactor.callLater( self.timeout, self.doTimeout) if self.maxTime: self.maxTimeoutTimer = self._reactor.callLater( self.maxTime, self.doMaxTimeout) for w in self.logFileWatchers: w.start()
def _startCommand(self): # ensure workdir exists if not os.path.isdir(self.workdir): os.makedirs(self.workdir) log.msg("RunProcess._startCommand") self.pp = RunProcessPP(self) self.using_comspec = False self.command = unicode2bytes(self.command, encoding=self.builder.unicode_encoding) if isinstance(self.command, bytes): if runtime.platformType == 'win32': # allow %COMSPEC% to have args argv = os.environ['COMSPEC'].split() if '/c' not in argv: argv += ['/c'] argv += [self.command] self.using_comspec = True else: # for posix, use /bin/sh. for other non-posix, well, doesn't # hurt to try argv = [b'/bin/sh', b'-c', self.command] display = self.fake_command else: # On windows, CreateProcess requires an absolute path to the executable. # When we call spawnProcess below, we pass argv[0] as the executable. # So, for .exe's that we have absolute paths to, we can call directly # Otherwise, we should run under COMSPEC (usually cmd.exe) to # handle path searching, etc. if (runtime.platformType == 'win32' and not (bytes2unicode(self.command[0], self.builder.unicode_encoding).lower().endswith(".exe") and os.path.isabs(self.command[0]))): # allow %COMSPEC% to have args argv = os.environ['COMSPEC'].split() if '/c' not in argv: argv += ['/c'] argv += list(self.command) self.using_comspec = True else: argv = self.command # Attempt to format this for use by a shell, although the process # isn't perfect display = shell_quote(self.fake_command, self.builder.unicode_encoding) display = bytes2unicode(display, self.builder.unicode_encoding) # $PWD usually indicates the current directory; spawnProcess may not # update this value, though, so we set it explicitly here. This causes # weird problems (bug #456) on msys, though.. if not self.environ.get('MACHTYPE', None) == 'i686-pc-msys': self.environ['PWD'] = os.path.abspath(self.workdir) # self.stdin is handled in RunProcessPP.connectionMade log.msg(u" " + display) self._addToBuffers(u'header', display + u"\n") # then comes the secondary information msg = u" in dir {0}".format(self.workdir) if self.timeout: if self.timeout == 1: unit = u"sec" else: unit = u"secs" msg += u" (timeout {0} {1})".format(self.timeout, unit) if self.maxTime: if self.maxTime == 1: unit = u"sec" else: unit = u"secs" msg += u" (maxTime {0} {1})".format(self.maxTime, unit) log.msg(u" " + msg) self._addToBuffers(u'header', msg + u"\n") msg = " watching logfiles {0}".format(self.logfiles) log.msg(" " + msg) self._addToBuffers('header', msg + u"\n") # then the obfuscated command array for resolving unambiguity msg = u" argv: {0}".format(self.fake_command) log.msg(u" " + msg) self._addToBuffers('header', msg + u"\n") # then the environment, since it sometimes causes problems if self.logEnviron: msg = u" environment:\n" env_names = sorted(self.environ.keys()) for name in env_names: msg += u" {0}={1}\n".format(bytes2unicode(name, encoding=self.builder.unicode_encoding), bytes2unicode(self.environ[name], encoding=self.builder.unicode_encoding)) log.msg(u" environment:\n{0}".format(pprint.pformat(self.environ))) self._addToBuffers(u'header', msg) if self.initialStdin: msg = u" writing {0} bytes to stdin".format(len(self.initialStdin)) log.msg(u" " + msg) self._addToBuffers(u'header', msg + u"\n") msg = u" using PTY: {0}".format(bool(self.usePTY)) log.msg(u" " + msg) self._addToBuffers(u'header', msg + u"\n") # put data into stdin and close it, if necessary. This will be # buffered until connectionMade is called if self.initialStdin: self.pp.setStdin(self.initialStdin) self.startTime = util.now(self._reactor) # start the process self.process = self._spawnProcess( self.pp, argv[0], argv, self.environ, self.workdir, usePTY=self.usePTY) # set up timeouts if self.timeout: self.ioTimeoutTimer = self._reactor.callLater( self.timeout, self.doTimeout) if self.maxTime: self.maxTimeoutTimer = self._reactor.callLater( self.maxTime, self.doMaxTimeout) for w in self.logFileWatchers: w.start()
def quote(e): if not e: return u'""' e = bytes2unicode(e, unicode_encoding) return pipes.quote(e)