def __init__(self, buildmaster_host, port, name, passwd, basedir, keepalive, usePTY=None, keepaliveTimeout=None, umask=None, maxdelay=300, numcpus=None, unicode_encoding=None, allow_shutdown=None, maxRetries=None): # note: keepaliveTimeout is ignored, but preserved here for # backward-compatibility assert usePTY is None, "worker-side usePTY is not supported anymore" service.MultiService.__init__(self) WorkerBase.__init__(self, name, basedir, umask=umask, unicode_encoding=unicode_encoding) if keepalive == 0: keepalive = None name = unicode2bytes(name, self.bot.unicode_encoding) passwd = unicode2bytes(passwd, self.bot.unicode_encoding) self.numcpus = numcpus self.shutdown_loop = None self.maxRetries = maxRetries if allow_shutdown == 'signal': if not hasattr(signal, 'SIGHUP'): raise ValueError("Can't install signal handler") elif allow_shutdown == 'file': self.shutdown_file = os.path.join(basedir, 'shutdown.stamp') self.shutdown_mtime = 0 self.allow_shutdown = allow_shutdown bf = self.bf = BotFactory(buildmaster_host, port, keepalive, maxdelay.bit_length, maxRetries=self.maxRetries, maxRetriesCallback=self.gracefulShutdown) bf.startLogin(credentials.UsernamePassword(name, passwd), client=self.bot) self.connection = c = internet.TCPClient( buildmaster_host, port, HangCheckFactory(bf, hung_callback=self._hung_connection)) c.setServiceParent(self)
def __init__(self, buildmaster_host, port, name, passwd, basedir, keepalive, usePTY=None, keepaliveTimeout=None, umask=None, maxdelay=None, numcpus=None, unicode_encoding=None, allow_shutdown=None, maxRetries=None, connection_string=None): assert usePTY is None, "worker-side usePTY is not supported anymore" assert (connection_string is None or (buildmaster_host, port) == (None, None)), ( "If you want to supply a connection string, " "then set host and port to None") service.MultiService.__init__(self) WorkerBase.__init__( self, name, basedir, umask=umask, unicode_encoding=unicode_encoding) if keepalive == 0: keepalive = None name = unicode2bytes(name, self.bot.unicode_encoding) passwd = unicode2bytes(passwd, self.bot.unicode_encoding) self.numcpus = numcpus self.shutdown_loop = None if allow_shutdown == 'signal': if not hasattr(signal, 'SIGHUP'): raise ValueError("Can't install signal handler") elif allow_shutdown == 'file': self.shutdown_file = os.path.join(basedir, 'shutdown.stamp') self.shutdown_mtime = 0 self.allow_shutdown = allow_shutdown bf = self.bf = BotFactory(buildmaster_host, port, keepalive, maxdelay) bf.startLogin( credentials.UsernamePassword(name, passwd), client=self.bot) if connection_string is None: connection_string = 'tcp:host={}:port={}'.format( buildmaster_host.replace(':', r'\:'), # escape ipv6 addresses port) endpoint = clientFromString(reactor, connection_string) def policy(attempt): if maxRetries and attempt >= maxRetries: reactor.stop() return backoffPolicy()(attempt) pb_service = ClientService(endpoint, bf, retryPolicy=policy) self.addService(pb_service)
def __init__(self, buildmaster_host, port, name, passwd, basedir, keepalive, usePTY=None, keepaliveTimeout=None, umask=None, maxdelay=300, numcpus=None, unicode_encoding=None, allow_shutdown=None, maxRetries=None): # note: keepaliveTimeout is ignored, but preserved here for # backward-compatibility assert usePTY is None, "worker-side usePTY is not supported anymore" service.MultiService.__init__(self) WorkerBase.__init__( self, name, basedir, umask=umask, unicode_encoding=unicode_encoding) if keepalive == 0: keepalive = None name = unicode2bytes(name, self.bot.unicode_encoding) passwd = unicode2bytes(passwd, self.bot.unicode_encoding) self.numcpus = numcpus self.shutdown_loop = None self.maxRetries = maxRetries if allow_shutdown == 'signal': if not hasattr(signal, 'SIGHUP'): raise ValueError("Can't install signal handler") elif allow_shutdown == 'file': self.shutdown_file = os.path.join(basedir, 'shutdown.stamp') self.shutdown_mtime = 0 self.allow_shutdown = allow_shutdown bf = self.bf = BotFactory(buildmaster_host, port, keepalive, maxdelay, maxRetries=self.maxRetries, maxRetriesCallback=self.gracefulShutdown) bf.startLogin( credentials.UsernamePassword(name, passwd), client=self.bot) self.connection = c = internet.TCPClient( buildmaster_host, port, HangCheckFactory(bf, hung_callback=self._hung_connection)) c.setServiceParent(self)
def _startCommand(self): # ensure workdir exists if not os.path.isdir(self.workdir): os.makedirs(self.workdir) log.msg("RunProcess._startCommand") self.pp = RunProcessPP(self) self.using_comspec = False self.command = unicode2bytes(self.command, encoding=self.builder.unicode_encoding) if isinstance(self.command, bytes): if runtime.platformType == 'win32': # allow %COMSPEC% to have args argv = os.environ['COMSPEC'].split() if '/c' not in argv: argv += ['/c'] argv += [self.command] self.using_comspec = True else: # for posix, use /bin/sh. for other non-posix, well, doesn't # hurt to try argv = [b'/bin/sh', b'-c', self.command] display = self.fake_command else: # On windows, CreateProcess requires an absolute path to the executable. # When we call spawnProcess below, we pass argv[0] as the executable. # So, for .exe's that we have absolute paths to, we can call directly # Otherwise, we should run under COMSPEC (usually cmd.exe) to # handle path searching, etc. if (runtime.platformType == 'win32' and not (bytes2unicode( self.command[0], self.builder.unicode_encoding).lower().endswith(".exe") and os.path.isabs(self.command[0]))): # allow %COMSPEC% to have args argv = os.environ['COMSPEC'].split() if '/c' not in argv: argv += ['/c'] argv += list(self.command) self.using_comspec = True else: argv = self.command # Attempt to format this for use by a shell, although the process # isn't perfect display = shell_quote(self.fake_command, self.builder.unicode_encoding) display = bytes2unicode(display, self.builder.unicode_encoding) # $PWD usually indicates the current directory; spawnProcess may not # update this value, though, so we set it explicitly here. This causes # weird problems (bug #456) on msys, though.. if not self.environ.get('MACHTYPE', None) == 'i686-pc-msys': self.environ['PWD'] = os.path.abspath(self.workdir) # self.stdin is handled in RunProcessPP.connectionMade log.msg(u" " + display) self._addToBuffers(u'header', display + u"\n") # then comes the secondary information msg = u" in dir {0}".format(self.workdir) if self.timeout: if self.timeout == 1: unit = u"sec" else: unit = u"secs" msg += u" (timeout {0} {1})".format(self.timeout, unit) if self.maxTime: if self.maxTime == 1: unit = u"sec" else: unit = u"secs" msg += u" (maxTime {0} {1})".format(self.maxTime, unit) log.msg(u" " + msg) self._addToBuffers(u'header', msg + u"\n") msg = " watching logfiles {0}".format(self.logfiles) log.msg(" " + msg) self._addToBuffers('header', msg + u"\n") # then the obfuscated command array for resolving unambiguity msg = u" argv: {0}".format(self.fake_command) log.msg(u" " + msg) self._addToBuffers('header', msg + u"\n") # then the environment, since it sometimes causes problems if self.logEnviron: msg = u" environment:\n" env_names = sorted(self.environ.keys()) for name in env_names: msg += u" {0}={1}\n".format( bytes2unicode(name, encoding=self.builder.unicode_encoding), bytes2unicode(self.environ[name], encoding=self.builder.unicode_encoding)) log.msg(u" environment:\n{0}".format(pprint.pformat(self.environ))) self._addToBuffers(u'header', msg) if self.initialStdin: msg = u" writing {0} bytes to stdin".format(len(self.initialStdin)) log.msg(u" " + msg) self._addToBuffers(u'header', msg + u"\n") msg = u" using PTY: {0}".format(bool(self.usePTY)) log.msg(u" " + msg) self._addToBuffers(u'header', msg + u"\n") # put data into stdin and close it, if necessary. This will be # buffered until connectionMade is called if self.initialStdin: self.pp.setStdin(self.initialStdin) self.startTime = util.now(self._reactor) # start the process self.process = self._spawnProcess(self.pp, argv[0], argv, self.environ, self.workdir, usePTY=self.usePTY) # set up timeouts if self.timeout: self.ioTimeoutTimer = self._reactor.callLater( self.timeout, self.doTimeout) if self.maxTime: self.maxTimeoutTimer = self._reactor.callLater( self.maxTime, self.doMaxTimeout) for w in self.logFileWatchers: w.start()
def __init__(self, buildmaster_host, port, name, passwd, basedir, keepalive, usePTY=None, keepaliveTimeout=None, umask=None, maxdelay=None, numcpus=None, unicode_encoding=None, useTls=None, allow_shutdown=None, maxRetries=None, connection_string=None): assert usePTY is None, "worker-side usePTY is not supported anymore" assert (connection_string is None or (buildmaster_host, port) == (None, None)), ( "If you want to supply a connection string, " "then set host and port to None") service.MultiService.__init__(self) WorkerBase.__init__(self, name, basedir, umask=umask, unicode_encoding=unicode_encoding) if keepalive == 0: keepalive = None name = unicode2bytes(name, self.bot.unicode_encoding) passwd = unicode2bytes(passwd, self.bot.unicode_encoding) self.numcpus = numcpus self.shutdown_loop = None if allow_shutdown == 'signal': if not hasattr(signal, 'SIGHUP'): raise ValueError("Can't install signal handler") elif allow_shutdown == 'file': self.shutdown_file = os.path.join(basedir, 'shutdown.stamp') self.shutdown_mtime = 0 self.allow_shutdown = allow_shutdown bf = self.bf = BotFactory(buildmaster_host, port, keepalive, maxdelay) bf.startLogin(credentials.UsernamePassword(name, passwd), client=self.bot) if connection_string is None: if useTls: connection_type = 'tls' else: connection_type = 'tcp' connection_string = '{}:host={}:port={}'.format( connection_type, buildmaster_host.replace(':', r'\:'), # escape ipv6 addresses port) endpoint = clientFromString(reactor, connection_string) def policy(attempt): if maxRetries and attempt >= maxRetries: reactor.stop() return backoffPolicy()(attempt) pb_service = ClientService(endpoint, bf, retryPolicy=policy) self.addService(pb_service)
class LogWatcher(LineOnlyReceiver): POLL_INTERVAL = 0.1 TIMEOUT_DELAY = 10.0 delimiter = unicode2bytes(os.linesep) def __init__(self, logfile): self.logfile = logfile self.in_reconfig = False self.transport = FakeTransport() self.pp = TailProcess() self.pp.lw = self self.timer = None def start(self): # If the log file doesn't exist, create it now. if not os.path.exists(self.logfile): open(self.logfile, 'a').close() # return a Deferred that fires when the start process has # finished. It errbacks with TimeoutError if the finish line has not # been seen within 10 seconds, and with ReconfigError if the error # line was seen. If the logfile could not be opened, it errbacks with # an IOError. if platform.system().lower() == 'sunos' and os.path.exists( '/usr/xpg4/bin/tail'): tailBin = "/usr/xpg4/bin/tail" else: tailBin = "/usr/bin/tail" self.p = reactor.spawnProcess( self.pp, tailBin, ("tail", "-f", "-n", "0", self.logfile), env=os.environ, ) self.running = True d = defer.maybeDeferred(self._start) return d def _start(self): self.d = defer.Deferred() self.timer = reactor.callLater(self.TIMEOUT_DELAY, self.timeout) return self.d def timeout(self): self.timer = None e = WorkerTimeoutError() self.finished(Failure(e)) def finished(self, results): try: self.p.signalProcess("KILL") except error.ProcessExitedAlready: pass if self.timer: self.timer.cancel() self.timer = None self.running = False self.in_reconfig = False self.d.callback(results) def lineReceived(self, line): if not self.running: return None if b"Log opened." in line: self.in_reconfig = True if b"loading configuration from" in line: self.in_reconfig = True if self.in_reconfig: print(line) if b"message from master: attached" in line: return self.finished("buildbot-worker") return None
def outReceived(self, data): self.lw.dataReceived(unicode2bytes(data))
def _startCommand(self): # ensure workdir exists if not os.path.isdir(self.workdir): os.makedirs(self.workdir) log.msg("RunProcess._startCommand") self.pp = RunProcessPP(self) self.using_comspec = False self.command = unicode2bytes(self.command, encoding=self.builder.unicode_encoding) if isinstance(self.command, bytes): if runtime.platformType == 'win32': # allow %COMSPEC% to have args argv = os.environ['COMSPEC'].split() if '/c' not in argv: argv += ['/c'] argv += [self.command] self.using_comspec = True else: # for posix, use /bin/sh. for other non-posix, well, doesn't # hurt to try argv = [b'/bin/sh', b'-c', self.command] display = self.fake_command else: # On windows, CreateProcess requires an absolute path to the executable. # When we call spawnProcess below, we pass argv[0] as the executable. # So, for .exe's that we have absolute paths to, we can call directly # Otherwise, we should run under COMSPEC (usually cmd.exe) to # handle path searching, etc. if (runtime.platformType == 'win32' and not (bytes2unicode(self.command[0], self.builder.unicode_encoding).lower().endswith(".exe") and os.path.isabs(self.command[0]))): # allow %COMSPEC% to have args argv = os.environ['COMSPEC'].split() if '/c' not in argv: argv += ['/c'] argv += list(self.command) self.using_comspec = True else: argv = self.command # Attempt to format this for use by a shell, although the process # isn't perfect display = shell_quote(self.fake_command, self.builder.unicode_encoding) display = bytes2unicode(display, self.builder.unicode_encoding) # $PWD usually indicates the current directory; spawnProcess may not # update this value, though, so we set it explicitly here. This causes # weird problems (bug #456) on msys, though.. if not self.environ.get('MACHTYPE', None) == 'i686-pc-msys': self.environ['PWD'] = os.path.abspath(self.workdir) # self.stdin is handled in RunProcessPP.connectionMade log.msg(u" " + display) self._addToBuffers(u'header', display + u"\n") # then comes the secondary information msg = u" in dir {0}".format(self.workdir) if self.timeout: if self.timeout == 1: unit = u"sec" else: unit = u"secs" msg += u" (timeout {0} {1})".format(self.timeout, unit) if self.maxTime: if self.maxTime == 1: unit = u"sec" else: unit = u"secs" msg += u" (maxTime {0} {1})".format(self.maxTime, unit) log.msg(u" " + msg) self._addToBuffers(u'header', msg + u"\n") msg = " watching logfiles {0}".format(self.logfiles) log.msg(" " + msg) self._addToBuffers('header', msg + u"\n") # then the obfuscated command array for resolving unambiguity msg = u" argv: {0}".format(self.fake_command) log.msg(u" " + msg) self._addToBuffers('header', msg + u"\n") # then the environment, since it sometimes causes problems if self.logEnviron: msg = u" environment:\n" env_names = sorted(self.environ.keys()) for name in env_names: msg += u" {0}={1}\n".format(bytes2unicode(name, encoding=self.builder.unicode_encoding), bytes2unicode(self.environ[name], encoding=self.builder.unicode_encoding)) log.msg(u" environment:\n{0}".format(pprint.pformat(self.environ))) self._addToBuffers(u'header', msg) if self.initialStdin: msg = u" writing {0} bytes to stdin".format(len(self.initialStdin)) log.msg(u" " + msg) self._addToBuffers(u'header', msg + u"\n") msg = u" using PTY: {0}".format(bool(self.usePTY)) log.msg(u" " + msg) self._addToBuffers(u'header', msg + u"\n") # put data into stdin and close it, if necessary. This will be # buffered until connectionMade is called if self.initialStdin: self.pp.setStdin(self.initialStdin) self.startTime = util.now(self._reactor) # start the process self.process = self._spawnProcess( self.pp, argv[0], argv, self.environ, self.workdir, usePTY=self.usePTY) # set up timeouts if self.timeout: self.ioTimeoutTimer = self._reactor.callLater( self.timeout, self.doTimeout) if self.maxTime: self.maxTimeoutTimer = self._reactor.callLater( self.maxTime, self.doMaxTimeout) for w in self.logFileWatchers: w.start()
def __init__(self, buildmaster_host, port, name, passwd, basedir, keepalive, usePTY=None, keepaliveTimeout=None, umask=None, maxdelay=None, numcpus=None, unicode_encoding=None, protocol='pb', useTls=None, allow_shutdown=None, maxRetries=None, connection_string=None, delete_leftover_dirs=False, proxy_connection_string=None): assert usePTY is None, "worker-side usePTY is not supported anymore" assert (connection_string is None or (buildmaster_host, port) == (None, None)), ( "If you want to supply a connection string, " "then set host and port to None") if protocol == 'pb': bot_class = BotPb elif protocol == 'msgpack_experimental_v1': if sys.version_info.major < 3: raise NotImplementedError( 'Msgpack protocol is not supported in Python2') bot_class = BotMsgpack else: raise ValueError('Unknown protocol {}'.format(protocol)) WorkerBase.__init__(self, name, basedir, bot_class, umask=umask, unicode_encoding=unicode_encoding, delete_leftover_dirs=delete_leftover_dirs) if keepalive == 0: keepalive = None name = unicode2bytes(name, self.bot.unicode_encoding) self.numcpus = numcpus self.shutdown_loop = None if allow_shutdown == 'signal': if not hasattr(signal, 'SIGHUP'): raise ValueError("Can't install signal handler") elif allow_shutdown == 'file': self.shutdown_file = os.path.join(basedir, 'shutdown.stamp') self.shutdown_mtime = 0 self.allow_shutdown = allow_shutdown if protocol == 'pb': passwd = unicode2bytes(passwd, self.bot.unicode_encoding) bf = self.bf = BotFactory(buildmaster_host, port, keepalive, maxdelay) bf.startLogin(credentials.UsernamePassword(name, passwd), client=self.bot) elif protocol == 'msgpack_experimental_v1': if connection_string is None: ws_conn_string = "ws://{}:{}".format(buildmaster_host, port) else: from urllib.parse import urlparse parsed_url = urlparse(connection_string) ws_conn_string = "ws://{}:{}".format(parsed_url.hostname, parsed_url.port) bf = self.bf = BuildbotWebSocketClientFactory(ws_conn_string) bf.protocol = BuildbotWebSocketClientProtocol self.bf.buildbot_bot = self.bot self.bf.name = name self.bf.password = passwd else: raise ValueError('Unknown protocol {}'.format(protocol)) def get_connection_string(host, port): if useTls: connection_type = 'tls' else: connection_type = 'tcp' return '{}:host={}:port={}'.format( connection_type, host.replace(':', r'\:'), # escape ipv6 addresses port) assert not (proxy_connection_string and connection_string), ( "If you want to use HTTP tunneling, then supply build master " "host and port rather than a connection string") if proxy_connection_string: log.msg("Using HTTP tunnel to connect through proxy") proxy_endpoint = clientFromString(reactor, proxy_connection_string) endpoint = HTTPTunnelEndpoint(buildmaster_host, port, proxy_endpoint) if useTls: from twisted.internet.endpoints import wrapClientTLS from twisted.internet.ssl import optionsForClientTLS contextFactory = optionsForClientTLS(hostname=buildmaster_host) endpoint = wrapClientTLS(contextFactory, endpoint) else: if connection_string is None: connection_string = get_connection_string( buildmaster_host, port) endpoint = clientFromString(reactor, connection_string) def policy(attempt): if maxRetries and attempt >= maxRetries: reactor.stop() return backoffPolicy()(attempt) pb_service = ClientService(endpoint, bf, retryPolicy=policy) self.addService(pb_service)