def setUp(self):
config = """
from buildbot.status import html
from buildbot.process.factory import BasicBuildFactory
from buildbot.buildslave import BuildSlave
f1 = BasicBuildFactory('cvsroot', 'cvsmodule')
BuildmasterConfig = {
'slaves': [BuildSlave('bot1', 'passwd1')],
'schedulers': [],
'builders': [{'name': 'builder1', 'slavename': 'bot1',
'builddir':'workdir', 'factory':f1}],
'slavePortnum': 0,
'status': [html.WebStatus(http_port=0)],
}
"""
if os.path.exists("test_logfile"):
shutil.rmtree("test_logfile")
os.mkdir("test_logfile")
self.master = m = ConfiguredMaster("test_logfile", config)
m.startService()
# hack to find out what randomly-assigned port it is listening on
port = self.find_webstatus(m).getPortnum()
self.port = port
# insert an event
req = base.BuildRequest("reason", sourcestamp.SourceStamp())
build1 = base.Build([req])
bs = m.status.getBuilder("builder1").newBuild()
bs.setReason("reason")
bs.buildStarted(build1)
step1 = BuildStep(name="setup")
step1.setBuild(build1)
bss = bs.addStepWithName("setup")
step1.setStepStatus(bss)
bss.stepStarted()
log1 = step1.addLog("output")
log1.addStdout("some stdout\n")
log1.finish()
log2 = step1.addHTMLLog("error", "<html>ouch</html>")
log3 = step1.addLog("big")
log3.addStdout("big log\n")
for i in range(1000):
log3.addStdout("a" * 500)
log3.addStderr("b" * 500)
log3.finish()
log4 = step1.addCompleteLog("bigcomplete",
"big2 log\n" + "a" * 1 * 1000 * 1000)
step1.step_status.stepFinished(builder.SUCCESS)
bs.buildFinished()
def setUp(self):
self.builder = FakeBuilder()
self.builder_status = builder.BuilderStatus("fakebuilder")
self.builder_status.basedir = "test_properties"
self.builder_status.nextBuildNumber = 5
rmdirRecursive(self.builder_status.basedir)
os.mkdir(self.builder_status.basedir)
self.build_status = self.builder_status.newBuild()
req = base.BuildRequest("reason",
SourceStamp(branch="branch2", revision=1234))
self.build = base.Build([req])
self.build.setBuilder(self.builder)
self.build.setupStatus(self.build_status)
self.build.setupSlaveBuilder(FakeSlaveBuilder())
def setUp(self):
rmtree("test_steps")
self.builder = FakeBuilder()
self.builder_status = builder.BuilderStatus("fakebuilder")
self.builder_status.basedir = "test_steps"
self.builder_status.nextBuildNumber = 0
os.mkdir(self.builder_status.basedir)
self.build_status = self.builder_status.newBuild()
req = base.BuildRequest("reason", SourceStamp(), 'test_builder')
self.build = base.Build([req])
self.build.build_status = self.build_status # fake it
self.build.builder = self.builder
self.build.slavebuilder = FakeSlaveBuilder()
self.remote = FakeRemote()
self.finished = 0
def _started(ign):
# hack to find out what randomly-assigned port it is listening on
port = self.find_webstatus(m).getPortnum()
self.port = port
# insert an event
req = BuildRequest("reason", sourcestamp.SourceStamp(), 'test_builder')
build1 = base.Build([req])
bs = m.status.getBuilder("builder1").newBuild()
bs.setReason("reason")
bs.buildStarted(build1)
step1 = BuildStep(name="setup")
step1.setBuild(build1)
bss = bs.addStepWithName("setup")
step1.setStepStatus(bss)
bss.stepStarted()
log1 = step1.addLog("output")
log1.addStdout(u"s\N{LATIN CAPITAL LETTER O WITH GRAVE}me stdout\n".encode("utf-8"))
log1.finish()
log2 = step1.addHTMLLog("error", "<html>ouch</html>")
del log2
log3 = step1.addLog("big")
log3.addStdout("big log\n")
for i in range(1000):
log3.addStdout("a" * 500)
log3.addStderr("b" * 500)
log3.finish()
log4 = step1.addCompleteLog("bigcomplete",
"big2 log\n" + "a" * 1*1000*1000)
del log4
log5 = step1.addLog("mixed")
log5.addHeader("header content")
log5.addStdout("this is stdout content")
log5.addStderr("errors go here")
log5.addEntry(5, "non-standard content on channel 5")
log5.addStderr(" and some trailing stderr")
d1 = defer.maybeDeferred(step1.step_status.stepFinished,
builder.SUCCESS)
bs.buildFinished()
return d1
def setUp(self):
self.builder = FakeBuilder()
self.builder_status = builder.BuilderStatus("fakebuilder")
self.basedir = self.mktemp()
os.mkdir(self.basedir)
self.builder_status.basedir = self.basedir
self.builder_status.nextBuildNumber = 5
self.build_status = self.builder_status.newBuild()
req = BuildRequest("reason",
SourceStamp(branch="branch2", revision="1234"),
'test_builder',
properties=Properties(scheduler="fakescheduler"))
self.build = base.Build([req])
self.build.build_status = self.build_status
self.build.setBuilder(self.builder)
self.build.setupProperties()
self.build.setupSlaveBuilder(FakeSlaveBuilder())
def setUp(self):
config = """
from buildbot.status import html
from buildbot.process.factory import BasicBuildFactory
from buildbot.buildslave import BuildSlave
from buildbot.config import BuilderConfig
f1 = BasicBuildFactory('cvsroot', 'cvsmodule')
BuildmasterConfig = c = {
'slaves': [BuildSlave('bot1', 'passwd1')],
'schedulers': [],
'slavePortnum': 0,
'status': [html.WebStatus(http_port=0)],
}
c['builders'] = [
BuilderConfig(name='builder1', slavename='bot1', factory=f1),
]
"""
if os.path.exists("test_logfile"):
shutil.rmtree("test_logfile")
os.mkdir("test_logfile")
self.master = m = ConfiguredMaster("test_logfile", config)
m.startService()
# hack to find out what randomly-assigned port it is listening on
port = self.find_webstatus(m).getPortnum()
self.port = port
# insert an event
req = base.BuildRequest("reason", sourcestamp.SourceStamp(),
'test_builder')
build1 = base.Build([req])
bs = m.status.getBuilder("builder1").newBuild()
bs.setReason("reason")
bs.buildStarted(build1)
step1 = BuildStep(name="setup")
step1.setBuild(build1)
bss = bs.addStepWithName("setup")
step1.setStepStatus(bss)
bss.stepStarted()
log1 = step1.addLog("output")
log1.addStdout(u"sÒme stdout\n")
log1.finish()
log2 = step1.addHTMLLog("error", "<html>ouch</html>")
log3 = step1.addLog("big")
log3.addStdout("big log\n")
for i in range(1000):
log3.addStdout("a" * 500)
log3.addStderr("b" * 500)
log3.finish()
log4 = step1.addCompleteLog("bigcomplete",
"big2 log\n" + "a" * 1 * 1000 * 1000)
log5 = step1.addLog("mixed")
log5.addHeader("header content")
log5.addStdout("this is stdout content")
log5.addStderr("errors go here")
log5.addEntry(5, "non-standard content on channel 5")
log5.addStderr(" and some trailing stderr")
d = defer.maybeDeferred(step1.step_status.stepFinished,
builder.SUCCESS)
bs.buildFinished()
return d
def MockBuild(my_builder,
buildsetup,
mastername,
slavename,
basepath=None,
build_properties=None,
slavedir=None):
"""Given a builder object and configuration, mock a Buildbot setup around it.
This sets up a mock BuildMaster, BuildSlave, Build, BuildStatus, and all other
superstructure required for BuildSteps inside the provided builder to render
properly. These BuildSteps are returned to the user in an array. It
additionally returns the build object (in order to get its properties if
desired).
buildsetup is passed straight into the FakeSource's init method and
contains sourcestamp information (revision, branch, etc).
basepath is the directory of the build (what goes under build/slave/, for
example 'Chromium_Linux_Builder'. It is nominally inferred from the builder
name, but it can be overridden. This is useful when pointing the buildrunner
at a different builder than what it's running under.
build_properties will update and override build_properties after all
builder-derived defaults have been set.
"""
my_factory = my_builder['factory']
steplist = ListSteps(my_factory)
build = base.Build([FakeRequest(buildsetup)])
safename = buildbot.util.safeTranslate(my_builder['name'])
my_builder['builddir'] = safename
my_builder.setdefault('slavebuilddir', safename)
workdir_root = None
if not slavedir:
workdir_root = os.path.join(SCRIPT_DIR, '..', '..', 'slave',
my_builder['slavebuilddir'])
if not basepath: basepath = safename
if not slavedir: slavedir = os.path.join(SCRIPT_DIR, '..', '..', 'slave')
basedir = os.path.join(slavedir, basepath)
build.basedir = basedir
if not workdir_root:
workdir_root = basedir
builderstatus = builder.BuilderStatus('test')
builderstatus.basedir = basedir
buildnumber = build_properties.get('buildnumber', 1)
builderstatus.nextBuildNumber = buildnumber + 1
mybuilder = real_builder.Builder(my_builder, builderstatus)
build.setBuilder(mybuilder)
build_status = build_module.BuildStatus(builderstatus, buildnumber)
build_status.setProperty('blamelist', [], 'Build')
build_status.setProperty('mastername', mastername, 'Build')
build_status.setProperty('slavename', slavename, 'Build')
build_status.setProperty('gtest_filter', [], 'Build')
build_status.setProperty('extra_args', [], 'Build')
build_status.setProperty('build_id', buildnumber, 'Build')
# if build_properties are passed in, overwrite the defaults above:
buildprops = Properties()
if build_properties:
buildprops.update(build_properties, 'Botmaster')
mybuilder.setBotmaster(FakeBotmaster(mastername, buildprops))
buildslave = FakeSlave(safename, my_builder.get('slavebuilddir'),
slavename)
build.build_status = build_status
build.setupSlaveBuilder(buildslave)
build.setupProperties()
process_steps(steplist, build, buildslave, build_status, workdir_root)
return steplist, build
def create_status(self):
for i in range(5):
if i % 2 == 0:
branch = "release"
else:
branch = None
c = Change("user", ["foo.c"] * i,
"see ticket #%i" % i,
revision=str(42 + i),
when=0.1 * i,
branch=branch)
self.master.change_svc.addChange(c)
ss = sourcestamp.SourceStamp(revision=42)
req = BuildRequest("reason", ss, 'test_builder')
build1 = base.Build([req])
bs = self.master.status.getBuilder("builder1").newBuild()
bs.setReason("reason")
bs.buildStarted(build1)
bs.setSourceStamp(ss)
bs.setProperty("revision", "42", "testcase")
bs.setProperty("got_revision", "47", "testcase")
bs.setProperty("branch", "release", "testcase")
step1 = BuildStep(name="setup")
step1.setBuild(build1)
bss = bs.addStepWithName("setup")
step1.setStepStatus(bss)
bss.stepStarted()
step2 = BuildStep(name="build")
step2.setBuild(build1)
bss = bs.addStepWithName("build")
step2.setStepStatus(bss)
bss.stepStarted()
step1.addURL("url1", "http://logurl.net/1")
step1.addURL("url2", "http://logurl.net/2")
step1.addURL("url3", "http://logurl.net/3")
log1 = step1.addLog("output")
log1.addStdout(
u"some stdout\n") # FIXME: Unicode here fails validation
log1.finish()
# this has to validate too for the test to pass
log2 = step1.addHTMLLog(
"error", '''
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html>
<head><title>Teh Log</title></head>
<body>Aaaiight</body>
</html>''')
del log2
log3 = step1.addLog("big")
log3.addStdout("somewhat big log\n")
for i in range(50):
log3.addStdout("a" * 50 + "\n")
log3.addStderr("b" * 50 + "\n")
log3.finish()
log5 = step1.addLog("mixed")
log5.addHeader("header content")
log5.addStdout("this is stdout content")
log5.addStderr("errors go here")
log5.addEntry(5, "non-standard content on channel 5")
log5.addStderr(" and some trailing stderr")
d = defer.succeed(None)
for i in range(1, 3):
ss = sourcestamp.SourceStamp(revision=42 + i, branch='release')
req = BuildRequest("reason", ss, 'test_builder')
build = base.Build([req] * i)
bs = self.master.status.getBuilder("builder%i" % i).newBuild()
bs.setReason("reason")
bs.buildStarted(build)
bs.setSourceStamp(ss)
s = BuildStep(name="setup")
s.setBuild(build1)
bss = bs.addStepWithName("setup")
s.setStepStatus(bss)
bss.stepStarted()
log = s.addLog("stdio")
log.finish()
d.chainDeferred(
defer.maybeDeferred(step1.step_status.stepFinished,
builder.SUCCESS))
bs.buildFinished()
return d
def main(args):
if args.list_masters:
masterpairs = get_masters()
pp_masters(masterpairs)
return 0
if args.master_dir:
config = read_config(args.master_dir, args.master_cfg)
else:
path = choose_master(args.mastername)
if not path:
return 2
config = read_config(path, args.master_cfg)
if not config:
return 2
mastername = config['BuildmasterConfig']['properties']['mastername']
builders = dup_slaves(config['BuildmasterConfig']['builders'])
if args.list_builders:
pp_builders(builders, mastername)
return 0
my_builder = choose(builders, args.spec)
if args.spec and 'hostname' in args.spec:
slavename = args.spec['hostname']
elif (args.spec and 'either'
in args.spec) and (args.spec['either'] != my_builder['name']):
slavename = args.spec['either']
else:
slavename = my_builder['slavename']
if not my_builder:
return 2
my_factory = my_builder['factory']
steplist = generate_steplist(my_factory)
if args.list_steps:
print
print 'listing steps in %s/%s:' % (mastername, my_builder['name'])
print
for step in steplist:
if hasattr(args,
'step_regex') and not args.step_regex.search(step.name):
print '-', step.name, '[skipped]'
elif hasattr(args,
'stepreject_regex') and (args.stepreject_regex.search(
step.name)):
print '-', step.name, '[skipped]'
else:
print '*', step.name
return 0
if not args.annotate:
print >> sys.stderr, 'using %s builder \'%s\'' % (mastername,
my_builder['name'])
if args.build_properties:
buildsetup = args.build_properties
else:
buildsetup = {}
buildsetup['revision'] = '%d' % args.revision
buildsetup['branch'] = 'src'
build = base.Build([FakeRequest(buildsetup)])
safename = buildbot.util.safeTranslate(my_builder['name'])
if hasattr(args, 'builderpath'):
basepath = args.builderpath
else:
basepath = safename
basedir = os.path.join('..', '..', '..', 'slave', basepath)
build.basedir = basedir
builderstatus = builder.BuilderStatus('test')
builderstatus.nextBuildNumber = 2
builderstatus.basedir = basedir
my_builder['builddir'] = safename
my_builder['slavebuilddir'] = safename
mybuilder = real_builder.Builder(my_builder, builderstatus)
build.setBuilder(mybuilder)
build_status = build_module.BuildStatus(builderstatus, 1)
build_status.setProperty('blamelist', [], 'Build')
build_status.setProperty('mastername', mastername, 'Build')
build_status.setProperty('slavename', slavename, 'Build')
build_status.setProperty('gtest_filter', [], 'Build')
# if build_properties are set on the CLI, overwrite the defaults
# set above when build.setupProperties is called
buildprops = Properties()
if args.build_properties:
buildprops.update(args.build_properties, 'Botmaster')
mybuilder.setBotmaster(FakeBotmaster(mastername, buildprops))
mylogger = LogClass(args.log)
buildslave = FakeSlave(safename, slavename)
buildslave.addUpdateAction(mylogger.log_to_file)
build.build_status = build_status
build.setupSlaveBuilder(buildslave)
build.setupProperties()
if args.output_build_properties:
print
print 'build properties:'
print propertiesToJSON(build.getProperties())
if args.output_factory_properties:
print
print 'factory properties:'
print propertiesToJSON(my_factory.properties)
if args.output_build_properties or args.output_factory_properties:
return 0
process_steps(steplist, build, buildslave, build_status, basedir)
commands = get_commands(steplist)
run_status = ReturnStatus()
start_time = time.clock()
commands_executed = 0
for command in commands:
if hasattr(args, 'step_regex'):
if not args.step_regex.search(command['name']):
if not args.annotate:
print >> sys.stderr, 'skipping step: ' + command['name']
continue
if hasattr(args, 'stepreject_regex'):
if args.stepreject_regex.search(command['name']):
if not args.annotate:
print >> sys.stderr, 'skipping step: ' + command['name']
continue
if not args.annotate:
print >> sys.stderr, 'running step: %s' % command['name']
else:
print '@@@BUILD_STEP %s@@@' % command['name']
print >> args.log, '(in %s): %s' % (command['workdir'],
shell_quote(command['command']))
mydir = os.getcwd()
myenv = os.environ
os.chdir(command['workdir'])
# python docs says this might cause leaks on FreeBSD/OSX
for envar in command['env']:
os.environ[envar] = command['env'][envar]
ret = chromium_utils.RunCommand(command['command'],
filter_obj=mylogger,
print_cmd=False)
os.chdir(mydir)
os.environ = myenv
commands_executed += 1
if ret != 0:
return 2
end_time = time.clock()
if not args.annotate:
print >> sys.stderr, '%d commands completed (%0.2fs).' % (
commands_executed, end_time - start_time)
else:
if commands_executed < 1:
print '0 commands executed.'
return run_status.code
