def __init__(self,
buildmaster_host,
port,
name,
passwd,
basedir,
keepalive,
usePTY=None,
keepaliveTimeout=None,
umask=None,
maxdelay=300,
numcpus=None,
unicode_encoding=None,
allow_shutdown=None,
maxRetries=None):
# note: keepaliveTimeout is ignored, but preserved here for
# backward-compatibility
assert usePTY is None, "worker-side usePTY is not supported anymore"
service.MultiService.__init__(self)
WorkerBase.__init__(self,
name,
basedir,
umask=umask,
unicode_encoding=unicode_encoding)
if keepalive == 0:
keepalive = None
name = unicode2bytes(name, self.bot.unicode_encoding)
passwd = unicode2bytes(passwd, self.bot.unicode_encoding)
self.numcpus = numcpus
self.shutdown_loop = None
self.maxRetries = maxRetries
if allow_shutdown == 'signal':
if not hasattr(signal, 'SIGHUP'):
raise ValueError("Can't install signal handler")
elif allow_shutdown == 'file':
self.shutdown_file = os.path.join(basedir, 'shutdown.stamp')
self.shutdown_mtime = 0
self.allow_shutdown = allow_shutdown
bf = self.bf = BotFactory(buildmaster_host,
port,
keepalive,
maxdelay.bit_length,
maxRetries=self.maxRetries,
maxRetriesCallback=self.gracefulShutdown)
bf.startLogin(credentials.UsernamePassword(name, passwd),
client=self.bot)
self.connection = c = internet.TCPClient(
buildmaster_host, port,
HangCheckFactory(bf, hung_callback=self._hung_connection))
c.setServiceParent(self)
def __init__(self, buildmaster_host, port, name, passwd, basedir,
keepalive, usePTY=None, keepaliveTimeout=None, umask=None,
maxdelay=None, numcpus=None, unicode_encoding=None,
allow_shutdown=None, maxRetries=None, connection_string=None):
assert usePTY is None, "worker-side usePTY is not supported anymore"
assert (connection_string is None or
(buildmaster_host, port) == (None, None)), (
"If you want to supply a connection string, "
"then set host and port to None")
service.MultiService.__init__(self)
WorkerBase.__init__(
self, name, basedir, umask=umask, unicode_encoding=unicode_encoding)
if keepalive == 0:
keepalive = None
name = unicode2bytes(name, self.bot.unicode_encoding)
passwd = unicode2bytes(passwd, self.bot.unicode_encoding)
self.numcpus = numcpus
self.shutdown_loop = None
if allow_shutdown == 'signal':
if not hasattr(signal, 'SIGHUP'):
raise ValueError("Can't install signal handler")
elif allow_shutdown == 'file':
self.shutdown_file = os.path.join(basedir, 'shutdown.stamp')
self.shutdown_mtime = 0
self.allow_shutdown = allow_shutdown
bf = self.bf = BotFactory(buildmaster_host, port, keepalive, maxdelay)
bf.startLogin(
credentials.UsernamePassword(name, passwd), client=self.bot)
if connection_string is None:
connection_string = 'tcp:host={}:port={}'.format(
buildmaster_host.replace(':', r'\:'), # escape ipv6 addresses
port)
endpoint = clientFromString(reactor, connection_string)
def policy(attempt):
if maxRetries and attempt >= maxRetries:
reactor.stop()
return backoffPolicy()(attempt)
pb_service = ClientService(endpoint, bf,
retryPolicy=policy)
self.addService(pb_service)
def __init__(self, buildmaster_host, port, name, passwd, basedir,
keepalive, usePTY=None, keepaliveTimeout=None, umask=None,
maxdelay=300, numcpus=None, unicode_encoding=None,
allow_shutdown=None, maxRetries=None):
# note: keepaliveTimeout is ignored, but preserved here for
# backward-compatibility
assert usePTY is None, "worker-side usePTY is not supported anymore"
service.MultiService.__init__(self)
WorkerBase.__init__(
self, name, basedir, umask=umask, unicode_encoding=unicode_encoding)
if keepalive == 0:
keepalive = None
name = unicode2bytes(name, self.bot.unicode_encoding)
passwd = unicode2bytes(passwd, self.bot.unicode_encoding)
self.numcpus = numcpus
self.shutdown_loop = None
self.maxRetries = maxRetries
if allow_shutdown == 'signal':
if not hasattr(signal, 'SIGHUP'):
raise ValueError("Can't install signal handler")
elif allow_shutdown == 'file':
self.shutdown_file = os.path.join(basedir, 'shutdown.stamp')
self.shutdown_mtime = 0
self.allow_shutdown = allow_shutdown
bf = self.bf = BotFactory(buildmaster_host, port, keepalive, maxdelay,
maxRetries=self.maxRetries, maxRetriesCallback=self.gracefulShutdown)
bf.startLogin(
credentials.UsernamePassword(name, passwd), client=self.bot)
self.connection = c = internet.TCPClient(
buildmaster_host, port,
HangCheckFactory(bf, hung_callback=self._hung_connection))
c.setServiceParent(self)
def _startCommand(self):
# ensure workdir exists
if not os.path.isdir(self.workdir):
os.makedirs(self.workdir)
log.msg("RunProcess._startCommand")
self.pp = RunProcessPP(self)
self.using_comspec = False
self.command = unicode2bytes(self.command,
encoding=self.builder.unicode_encoding)
if isinstance(self.command, bytes):
if runtime.platformType == 'win32':
# allow %COMSPEC% to have args
argv = os.environ['COMSPEC'].split()
if '/c' not in argv:
argv += ['/c']
argv += [self.command]
self.using_comspec = True
else:
# for posix, use /bin/sh. for other non-posix, well, doesn't
# hurt to try
argv = [b'/bin/sh', b'-c', self.command]
display = self.fake_command
else:
# On windows, CreateProcess requires an absolute path to the executable.
# When we call spawnProcess below, we pass argv[0] as the executable.
# So, for .exe's that we have absolute paths to, we can call directly
# Otherwise, we should run under COMSPEC (usually cmd.exe) to
# handle path searching, etc.
if (runtime.platformType == 'win32'
and not (bytes2unicode(
self.command[0],
self.builder.unicode_encoding).lower().endswith(".exe")
and os.path.isabs(self.command[0]))):
# allow %COMSPEC% to have args
argv = os.environ['COMSPEC'].split()
if '/c' not in argv:
argv += ['/c']
argv += list(self.command)
self.using_comspec = True
else:
argv = self.command
# Attempt to format this for use by a shell, although the process
# isn't perfect
display = shell_quote(self.fake_command,
self.builder.unicode_encoding)
display = bytes2unicode(display, self.builder.unicode_encoding)
# $PWD usually indicates the current directory; spawnProcess may not
# update this value, though, so we set it explicitly here. This causes
# weird problems (bug #456) on msys, though..
if not self.environ.get('MACHTYPE', None) == 'i686-pc-msys':
self.environ['PWD'] = os.path.abspath(self.workdir)
# self.stdin is handled in RunProcessPP.connectionMade
log.msg(u" " + display)
self._addToBuffers(u'header', display + u"\n")
# then comes the secondary information
msg = u" in dir {0}".format(self.workdir)
if self.timeout:
if self.timeout == 1:
unit = u"sec"
else:
unit = u"secs"
msg += u" (timeout {0} {1})".format(self.timeout, unit)
if self.maxTime:
if self.maxTime == 1:
unit = u"sec"
else:
unit = u"secs"
msg += u" (maxTime {0} {1})".format(self.maxTime, unit)
log.msg(u" " + msg)
self._addToBuffers(u'header', msg + u"\n")
msg = " watching logfiles {0}".format(self.logfiles)
log.msg(" " + msg)
self._addToBuffers('header', msg + u"\n")
# then the obfuscated command array for resolving unambiguity
msg = u" argv: {0}".format(self.fake_command)
log.msg(u" " + msg)
self._addToBuffers('header', msg + u"\n")
# then the environment, since it sometimes causes problems
if self.logEnviron:
msg = u" environment:\n"
env_names = sorted(self.environ.keys())
for name in env_names:
msg += u" {0}={1}\n".format(
bytes2unicode(name,
encoding=self.builder.unicode_encoding),
bytes2unicode(self.environ[name],
encoding=self.builder.unicode_encoding))
log.msg(u" environment:\n{0}".format(pprint.pformat(self.environ)))
self._addToBuffers(u'header', msg)
if self.initialStdin:
msg = u" writing {0} bytes to stdin".format(len(self.initialStdin))
log.msg(u" " + msg)
self._addToBuffers(u'header', msg + u"\n")
msg = u" using PTY: {0}".format(bool(self.usePTY))
log.msg(u" " + msg)
self._addToBuffers(u'header', msg + u"\n")
# put data into stdin and close it, if necessary. This will be
# buffered until connectionMade is called
if self.initialStdin:
self.pp.setStdin(self.initialStdin)
self.startTime = util.now(self._reactor)
# start the process
self.process = self._spawnProcess(self.pp,
argv[0],
argv,
self.environ,
self.workdir,
usePTY=self.usePTY)
# set up timeouts
if self.timeout:
self.ioTimeoutTimer = self._reactor.callLater(
self.timeout, self.doTimeout)
if self.maxTime:
self.maxTimeoutTimer = self._reactor.callLater(
self.maxTime, self.doMaxTimeout)
for w in self.logFileWatchers:
w.start()
def __init__(self,
buildmaster_host,
port,
name,
passwd,
basedir,
keepalive,
usePTY=None,
keepaliveTimeout=None,
umask=None,
maxdelay=None,
numcpus=None,
unicode_encoding=None,
useTls=None,
allow_shutdown=None,
maxRetries=None,
connection_string=None):
assert usePTY is None, "worker-side usePTY is not supported anymore"
assert (connection_string is None
or (buildmaster_host, port) == (None, None)), (
"If you want to supply a connection string, "
"then set host and port to None")
service.MultiService.__init__(self)
WorkerBase.__init__(self,
name,
basedir,
umask=umask,
unicode_encoding=unicode_encoding)
if keepalive == 0:
keepalive = None
name = unicode2bytes(name, self.bot.unicode_encoding)
passwd = unicode2bytes(passwd, self.bot.unicode_encoding)
self.numcpus = numcpus
self.shutdown_loop = None
if allow_shutdown == 'signal':
if not hasattr(signal, 'SIGHUP'):
raise ValueError("Can't install signal handler")
elif allow_shutdown == 'file':
self.shutdown_file = os.path.join(basedir, 'shutdown.stamp')
self.shutdown_mtime = 0
self.allow_shutdown = allow_shutdown
bf = self.bf = BotFactory(buildmaster_host, port, keepalive, maxdelay)
bf.startLogin(credentials.UsernamePassword(name, passwd),
client=self.bot)
if connection_string is None:
if useTls:
connection_type = 'tls'
else:
connection_type = 'tcp'
connection_string = '{}:host={}:port={}'.format(
connection_type,
buildmaster_host.replace(':', r'\:'), # escape ipv6 addresses
port)
endpoint = clientFromString(reactor, connection_string)
def policy(attempt):
if maxRetries and attempt >= maxRetries:
reactor.stop()
return backoffPolicy()(attempt)
pb_service = ClientService(endpoint, bf, retryPolicy=policy)
self.addService(pb_service)
class LogWatcher(LineOnlyReceiver):
POLL_INTERVAL = 0.1
TIMEOUT_DELAY = 10.0
delimiter = unicode2bytes(os.linesep)
def __init__(self, logfile):
self.logfile = logfile
self.in_reconfig = False
self.transport = FakeTransport()
self.pp = TailProcess()
self.pp.lw = self
self.timer = None
def start(self):
# If the log file doesn't exist, create it now.
if not os.path.exists(self.logfile):
open(self.logfile, 'a').close()
# return a Deferred that fires when the start process has
# finished. It errbacks with TimeoutError if the finish line has not
# been seen within 10 seconds, and with ReconfigError if the error
# line was seen. If the logfile could not be opened, it errbacks with
# an IOError.
if platform.system().lower() == 'sunos' and os.path.exists(
'/usr/xpg4/bin/tail'):
tailBin = "/usr/xpg4/bin/tail"
else:
tailBin = "/usr/bin/tail"
self.p = reactor.spawnProcess(
self.pp,
tailBin,
("tail", "-f", "-n", "0", self.logfile),
env=os.environ,
)
self.running = True
d = defer.maybeDeferred(self._start)
return d
def _start(self):
self.d = defer.Deferred()
self.timer = reactor.callLater(self.TIMEOUT_DELAY, self.timeout)
return self.d
def timeout(self):
self.timer = None
e = WorkerTimeoutError()
self.finished(Failure(e))
def finished(self, results):
try:
self.p.signalProcess("KILL")
except error.ProcessExitedAlready:
pass
if self.timer:
self.timer.cancel()
self.timer = None
self.running = False
self.in_reconfig = False
self.d.callback(results)
def lineReceived(self, line):
if not self.running:
return None
if b"Log opened." in line:
self.in_reconfig = True
if b"loading configuration from" in line:
self.in_reconfig = True
if self.in_reconfig:
print(line)
if b"message from master: attached" in line:
return self.finished("buildbot-worker")
return None
def outReceived(self, data):
self.lw.dataReceived(unicode2bytes(data))
def _startCommand(self):
# ensure workdir exists
if not os.path.isdir(self.workdir):
os.makedirs(self.workdir)
log.msg("RunProcess._startCommand")
self.pp = RunProcessPP(self)
self.using_comspec = False
self.command = unicode2bytes(self.command, encoding=self.builder.unicode_encoding)
if isinstance(self.command, bytes):
if runtime.platformType == 'win32':
# allow %COMSPEC% to have args
argv = os.environ['COMSPEC'].split()
if '/c' not in argv:
argv += ['/c']
argv += [self.command]
self.using_comspec = True
else:
# for posix, use /bin/sh. for other non-posix, well, doesn't
# hurt to try
argv = [b'/bin/sh', b'-c', self.command]
display = self.fake_command
else:
# On windows, CreateProcess requires an absolute path to the executable.
# When we call spawnProcess below, we pass argv[0] as the executable.
# So, for .exe's that we have absolute paths to, we can call directly
# Otherwise, we should run under COMSPEC (usually cmd.exe) to
# handle path searching, etc.
if (runtime.platformType == 'win32' and
not (bytes2unicode(self.command[0],
self.builder.unicode_encoding).lower().endswith(".exe") and
os.path.isabs(self.command[0]))):
# allow %COMSPEC% to have args
argv = os.environ['COMSPEC'].split()
if '/c' not in argv:
argv += ['/c']
argv += list(self.command)
self.using_comspec = True
else:
argv = self.command
# Attempt to format this for use by a shell, although the process
# isn't perfect
display = shell_quote(self.fake_command, self.builder.unicode_encoding)
display = bytes2unicode(display, self.builder.unicode_encoding)
# $PWD usually indicates the current directory; spawnProcess may not
# update this value, though, so we set it explicitly here. This causes
# weird problems (bug #456) on msys, though..
if not self.environ.get('MACHTYPE', None) == 'i686-pc-msys':
self.environ['PWD'] = os.path.abspath(self.workdir)
# self.stdin is handled in RunProcessPP.connectionMade
log.msg(u" " + display)
self._addToBuffers(u'header', display + u"\n")
# then comes the secondary information
msg = u" in dir {0}".format(self.workdir)
if self.timeout:
if self.timeout == 1:
unit = u"sec"
else:
unit = u"secs"
msg += u" (timeout {0} {1})".format(self.timeout, unit)
if self.maxTime:
if self.maxTime == 1:
unit = u"sec"
else:
unit = u"secs"
msg += u" (maxTime {0} {1})".format(self.maxTime, unit)
log.msg(u" " + msg)
self._addToBuffers(u'header', msg + u"\n")
msg = " watching logfiles {0}".format(self.logfiles)
log.msg(" " + msg)
self._addToBuffers('header', msg + u"\n")
# then the obfuscated command array for resolving unambiguity
msg = u" argv: {0}".format(self.fake_command)
log.msg(u" " + msg)
self._addToBuffers('header', msg + u"\n")
# then the environment, since it sometimes causes problems
if self.logEnviron:
msg = u" environment:\n"
env_names = sorted(self.environ.keys())
for name in env_names:
msg += u" {0}={1}\n".format(bytes2unicode(name, encoding=self.builder.unicode_encoding),
bytes2unicode(self.environ[name], encoding=self.builder.unicode_encoding))
log.msg(u" environment:\n{0}".format(pprint.pformat(self.environ)))
self._addToBuffers(u'header', msg)
if self.initialStdin:
msg = u" writing {0} bytes to stdin".format(len(self.initialStdin))
log.msg(u" " + msg)
self._addToBuffers(u'header', msg + u"\n")
msg = u" using PTY: {0}".format(bool(self.usePTY))
log.msg(u" " + msg)
self._addToBuffers(u'header', msg + u"\n")
# put data into stdin and close it, if necessary. This will be
# buffered until connectionMade is called
if self.initialStdin:
self.pp.setStdin(self.initialStdin)
self.startTime = util.now(self._reactor)
# start the process
self.process = self._spawnProcess(
self.pp, argv[0], argv,
self.environ,
self.workdir,
usePTY=self.usePTY)
# set up timeouts
if self.timeout:
self.ioTimeoutTimer = self._reactor.callLater(
self.timeout, self.doTimeout)
if self.maxTime:
self.maxTimeoutTimer = self._reactor.callLater(
self.maxTime, self.doMaxTimeout)
for w in self.logFileWatchers:
w.start()
def outReceived(self, data):
self.lw.dataReceived(unicode2bytes(data))
def __init__(self,
buildmaster_host,
port,
name,
passwd,
basedir,
keepalive,
usePTY=None,
keepaliveTimeout=None,
umask=None,
maxdelay=None,
numcpus=None,
unicode_encoding=None,
protocol='pb',
useTls=None,
allow_shutdown=None,
maxRetries=None,
connection_string=None,
delete_leftover_dirs=False,
proxy_connection_string=None):
assert usePTY is None, "worker-side usePTY is not supported anymore"
assert (connection_string is None
or (buildmaster_host, port) == (None, None)), (
"If you want to supply a connection string, "
"then set host and port to None")
if protocol == 'pb':
bot_class = BotPb
elif protocol == 'msgpack_experimental_v1':
if sys.version_info.major < 3:
raise NotImplementedError(
'Msgpack protocol is not supported in Python2')
bot_class = BotMsgpack
else:
raise ValueError('Unknown protocol {}'.format(protocol))
WorkerBase.__init__(self,
name,
basedir,
bot_class,
umask=umask,
unicode_encoding=unicode_encoding,
delete_leftover_dirs=delete_leftover_dirs)
if keepalive == 0:
keepalive = None
name = unicode2bytes(name, self.bot.unicode_encoding)
self.numcpus = numcpus
self.shutdown_loop = None
if allow_shutdown == 'signal':
if not hasattr(signal, 'SIGHUP'):
raise ValueError("Can't install signal handler")
elif allow_shutdown == 'file':
self.shutdown_file = os.path.join(basedir, 'shutdown.stamp')
self.shutdown_mtime = 0
self.allow_shutdown = allow_shutdown
if protocol == 'pb':
passwd = unicode2bytes(passwd, self.bot.unicode_encoding)
bf = self.bf = BotFactory(buildmaster_host, port, keepalive,
maxdelay)
bf.startLogin(credentials.UsernamePassword(name, passwd),
client=self.bot)
elif protocol == 'msgpack_experimental_v1':
if connection_string is None:
ws_conn_string = "ws://{}:{}".format(buildmaster_host, port)
else:
from urllib.parse import urlparse
parsed_url = urlparse(connection_string)
ws_conn_string = "ws://{}:{}".format(parsed_url.hostname,
parsed_url.port)
bf = self.bf = BuildbotWebSocketClientFactory(ws_conn_string)
bf.protocol = BuildbotWebSocketClientProtocol
self.bf.buildbot_bot = self.bot
self.bf.name = name
self.bf.password = passwd
else:
raise ValueError('Unknown protocol {}'.format(protocol))
def get_connection_string(host, port):
if useTls:
connection_type = 'tls'
else:
connection_type = 'tcp'
return '{}:host={}:port={}'.format(
connection_type,
host.replace(':', r'\:'), # escape ipv6 addresses
port)
assert not (proxy_connection_string and connection_string), (
"If you want to use HTTP tunneling, then supply build master "
"host and port rather than a connection string")
if proxy_connection_string:
log.msg("Using HTTP tunnel to connect through proxy")
proxy_endpoint = clientFromString(reactor, proxy_connection_string)
endpoint = HTTPTunnelEndpoint(buildmaster_host, port,
proxy_endpoint)
if useTls:
from twisted.internet.endpoints import wrapClientTLS
from twisted.internet.ssl import optionsForClientTLS
contextFactory = optionsForClientTLS(hostname=buildmaster_host)
endpoint = wrapClientTLS(contextFactory, endpoint)
else:
if connection_string is None:
connection_string = get_connection_string(
buildmaster_host, port)
endpoint = clientFromString(reactor, connection_string)
def policy(attempt):
if maxRetries and attempt >= maxRetries:
reactor.stop()
return backoffPolicy()(attempt)
pb_service = ClientService(endpoint, bf, retryPolicy=policy)
self.addService(pb_service)
