def target_builder(arch, mode):
test_py = os.path.join('tools', 'test.py')
test_args = [
sys.executable, test_py, '--progress=line', '--report', '--time',
'--compiler=none', '--runtime=vm', '--write-debug-log',
'--write-test-outcome-log', '--mode=' + mode, '--arch=' + arch,
'--exclude-suite=pkg'
]
revision = int(os.environ['BUILDBOT_GOT_REVISION'])
tarball = tarball_name(arch, mode, revision)
temporary_files = [tarball]
bot.Clobber()
try:
with bot.BuildStep('Fetch build tarball'):
run([GSUTIL, 'cp', "%s/%s" % (GCS_BUCKET, tarball), tarball])
with bot.BuildStep('Unpack build tarball'):
run(['tar', '-xjf', tarball])
with bot.BuildStep('execute tests'):
run(test_args)
with bot.BuildStep('execute checked_tests'):
run(test_args + ['--checked', '--append_logs'])
finally:
for path in temporary_files:
if os.path.exists(path):
os.remove(path)
# We always clobber this to save disk on the arm board.
bot.Clobber(force=True)
def cross_compiling_builder(arch, mode):
build_py = os.path.join('tools', 'build.py')
revision = int(os.environ['BUILDBOT_GOT_REVISION'])
tarball = tarball_name(arch, mode, revision)
temporary_files = [tarball]
bot.Clobber()
try:
num_run = int(os.environ['BUILDBOT_ANNOTATED_STEPS_RUN'])
if num_run == 1:
with bot.BuildStep('Build %s %s' % (arch, mode)):
run([sys.executable, build_py,
'-m%s' % mode, '--arch=%s' % arch])
with bot.BuildStep('Create build tarball'):
run(['tar', '-cjf', tarball, '--exclude=**/obj',
'--exclude=**/obj.host', '--exclude=**/obj.target',
'--exclude=**/*analyzer*', 'out/'])
with bot.BuildStep('Upload build tarball'):
uri = "%s/%s" % (GCS_BUCKET, tarball)
run([GSUTIL, 'cp', tarball, uri])
run([GSUTIL, 'setacl', 'public-read', uri])
elif num_run == 2:
with bot.BuildStep('tests'):
print "Please see the target device for results."
print "We no longer record/replay tests."
else:
raise Exception("Invalid annotated steps run")
finally:
for path in temporary_files:
if os.path.exists(path):
os.remove(path)
def StepsSDK(debug_log, system, modes, archs, embedded_libs):
no_clang = system == 'linux'
configurations = GetBuildConfigurations(system=system,
modes=modes,
archs=archs,
asans=[False],
no_clang=no_clang,
embedded_libs=embedded_libs,
use_sdks=[True])
bot.Clobber(force=True)
StepGyp()
cross_mode = 'release'
cross_arch = 'xarm'
cross_system = 'linux'
# We only cross compile on linux
if system == 'linux':
StepsCreateDebianPackage()
StepsArchiveDebianPackage()
CrossCompile(cross_system, [cross_mode], cross_arch)
StepsArchiveCrossCompileBundle(cross_mode, cross_arch)
StepsCreateArchiveRaspbianImge()
elif system == 'mac':
StepsGetArmBinaries(cross_mode, cross_arch)
StepsGetArmDeb()
# We currently only build documentation on linux.
StepsGetDocs()
for configuration in configurations:
StepBuild(configuration['build_conf'], configuration['build_dir'])
StepsBundleSDK(configuration['build_dir'], system)
StepsArchiveSDK(configuration['build_dir'], system,
configuration['mode'], configuration['arch'])
for configuration in configurations:
StepsTestSDK(debug_log, configuration)
StepsSanityChecking(configuration['build_dir'])
def StepsTargetRunner(debug_log, system, mode, arch):
"""This step downloads XARM build artifacts and runs tests.
The buildbot master only triggers this step once the `StepCrossBuilder` step
(defined above) has already been executed. This `StepsTargetRunner` can
therefore download/extract the build artifacts which were archived by
`StepCrossBuilder` and run the tests."""
revision = os.environ['BUILDBOT_GOT_REVISION']
tarball = TarballName(arch, revision)
try:
with bot.BuildStep('Fetch build tarball'):
Run([GSUTIL, 'cp', "%s/%s" % (GCS_BUCKET, tarball), tarball])
with bot.BuildStep('Unpack build tarball'):
Run(['tar', '-xjf', tarball])
# Run tests on all necessary configurations.
configurations = GetBuildConfigurations(system=system,
modes=[mode],
archs=[arch],
asans=[False],
embedded_libs=[False],
use_sdks=[False])
for snapshot_run in [True, False]:
for configuration in configurations:
if not ShouldSkipConfiguration(snapshot_run, configuration):
build_dir = configuration['build_dir']
# Sanity check we got build artifacts which we expect.
assert os.path.exists(os.path.join(build_dir,
'dartino-vm'))
# TODO(kustermann): This is hackisch, but our current copying of the
# dart binary makes this a requirement.
dart_arm = 'third_party/bin/linux/dart-arm'
destination = os.path.join(build_dir, 'dart')
shutil.copyfile(dart_arm, destination)
shutil.copymode(dart_arm, destination)
StepDisableAnalytics(build_dir)
def run():
StepTest(configuration=configuration,
snapshot_run=snapshot_run,
debug_log=debug_log)
#RunWithCoreDumpArchiving(run, build_dir, build_conf)
run()
finally:
if os.path.exists(tarball):
os.remove(tarball)
# We always clobber this to save disk on the arm board.
bot.Clobber(force=True)
def Main():
name, _ = bot.GetBotName()
dartino_match = re.match(DARTINO_REGEXP, name)
cross_match = re.match(CROSS_REGEXP, name)
target_match = re.match(TARGET_REGEXP, name)
if not dartino_match and not cross_match and not target_match:
raise Exception('Invalid buildername')
SetupClangEnvironment(utils.GuessOS())
SetupJavaEnvironment(utils.GuessOS())
# Clobber build directory if the checkbox was pressed on the BB.
with utils.ChangedWorkingDirectory(DARTINO_PATH):
bot.Clobber()
# Accumulate daemon logs messages in '.debug.log' to be displayed on the
# buildbot.Log
with utils.ChangedWorkingDirectory(DARTINO_PATH):
StepsCleanLogs()
with open(DEBUG_LOG, 'w') as debug_log:
if dartino_match:
system = dartino_match.group('system')
if system == 'lk':
StepsLK(debug_log)
elif system == 'free-rtos':
StepsFreeRtos(debug_log)
else:
modes = ['debug', 'release']
archs = ['ia32', 'x64']
asans = [False]
embedded_libs = [False]
# Split configurations?
partial_configuration =\
dartino_match.group('partial_configuration') != None
if partial_configuration:
architecture_match = dartino_match.group(
'architecture')
archs = {
'x86': ['ia32', 'x64'],
'x64': ['x64'],
'ia32': ['ia32'],
}[architecture_match]
modes = [dartino_match.group('mode')]
asans = [bool(dartino_match.group('asan'))]
embedded_libs = [
bool(dartino_match.group('embedded_libs'))
]
sdk_build = dartino_match.group('sdk')
if sdk_build:
StepsSDK(debug_log, system, modes, archs,
embedded_libs)
else:
StepsNormal(debug_log, system, modes, archs, asans,
embedded_libs)
elif cross_match:
system = cross_match.group(1)
arch = cross_match.group(2)
assert system == 'linux'
assert arch == 'arm'
modes = ['debug', 'release']
arch = 'xarm'
StepsCrossBuilder(debug_log, system, modes, arch)
elif target_match:
assert target_match.group(1) == 'linux'
system = 'linux'
mode = target_match.group(2)
arch = 'xarm'
StepsTargetRunner(debug_log, system, mode, arch)
StepsShowLogs()
return 1 if bot.HAS_FAILURES else 0
def StepsSDK(debug_log, system, modes, archs, embedded_libs):
no_clang = system == 'linux'
configurations = GetBuildConfigurations(system=system,
modes=modes,
archs=archs,
asans=[False],
no_clang=no_clang,
embedded_libs=embedded_libs,
use_sdks=[True])
bot.Clobber(force=True)
StepGyp()
cross_mode = 'release'
cross_archs = ['xarm', 'stm']
cross_system = 'linux'
# We only cross compile on linux
if system == 'linux':
StepsCreateDebianPackage()
StepsArchiveDebianPackage()
# We need the dartino daemon process to compile snapshots.
for arch in ['ia32', 'x64']:
host_configuration = GetBuildConfigurations(system=utils.GuessOS(),
modes=['release'],
archs=[arch],
asans=[False],
embedded_libs=[False],
use_sdks=[False])[0]
StepBuild(host_configuration['build_conf'],
host_configuration['build_dir'])
for cross_arch in cross_archs:
CrossCompile(cross_system, [cross_mode], cross_arch)
StepsArchiveCrossCompileBundle(cross_mode, cross_arch)
StepsCreateArchiveRaspbianImge()
elif system == 'mac':
# We need the 32-bit build for the dartino-flashify program.
if 'ia32' not in archs:
ia32_configuration = GetBuildConfigurations(
system=system,
modes=['release'],
archs=['ia32'],
asans=[False],
no_clang=no_clang,
embedded_libs=embedded_libs,
use_sdks=[True])[0]
StepBuild(ia32_configuration['build_conf'],
ia32_configuration['build_dir'])
for cross_arch in cross_archs:
StepsGetCrossBinaries(cross_mode, cross_arch)
StepsGetArmDeb()
# We currently only build documentation on linux.
StepsGetDocs()
for configuration in configurations:
StepBuild(configuration['build_conf'], configuration['build_dir'])
StepsBundleSDK(configuration['build_dir'], system)
StepsArchiveSDK(configuration['build_dir'], system,
configuration['mode'], configuration['arch'])
for configuration in configurations:
StepsTestSDK(debug_log, configuration)
StepsSanityChecking(configuration['build_dir'])
StepsArchiveGCCArmNoneEabi(system)
StepsArchiveOpenOCD(system)
gsutil.upload(local_path, remote_path, public=True)
if checksum_files:
# 'local_path' may have a different filename than 'remote_path'. So we need
# to make sure the *.md5sum file contains the correct name.
assert '/' in remote_path and not remote_path.endswith('/')
mangled_filename = remote_path[remote_path.rfind('/') + 1:]
local_md5sum = bot_utils.CreateMD5ChecksumFile(local_path,
mangled_filename)
gsutil.upload(local_md5sum, remote_path + '.md5sum', public=True)
local_sha256 = bot_utils.CreateSha256ChecksumFile(
local_path, mangled_filename)
gsutil.upload(local_sha256, remote_path + '.sha256sum', public=True)
def Run(command, env=None):
print "Running %s" % ' '.join(command)
print "Environment %s" % env
return bot.RunProcess(command, env=env)
if __name__ == '__main__':
# We always clobber the bot, to make sure releases are build from scratch
force = CHANNEL != bot_utils.Channel.BLEEDING_EDGE
bot.Clobber(force=force)
CreateUploadSDK()
if BUILD_OS == 'linux':
CreateUploadVersionFile()
CreateUploadAPIDocs()
def main():
name, is_buildbot = bot.GetBotName()
build_py = os.path.join('tools', 'build.py')
test_py = os.path.join('tools', 'test.py')
cross_vm_pattern_match = re.match(CROSS_VM, name)
target_vm_pattern_match = re.match(TARGET_VM, name)
if cross_vm_pattern_match:
arch = cross_vm_pattern_match.group(1)
mode = cross_vm_pattern_match.group(2)
bot.Clobber()
with bot.BuildStep('Build %s %s' % (arch, mode)):
args = [
sys.executable, build_py,
'-m%s' % mode,
'--arch=%s' % arch, 'runtime'
]
run(args)
tarball = 'cross_build_%s_%s.tar.bz2' % (arch, mode)
try:
with bot.BuildStep('Create build tarball'):
run([
'tar', '-cjf', tarball, '--exclude=**/obj',
'--exclude=**/obj.host', '--exclude=**/obj.target',
'--exclude=**/*analyzer*', 'out/'
])
with bot.BuildStep('Upload build tarball'):
uri = "%s/%s" % (GCS_BUCKET, tarball)
run([GSUTIL, 'cp', tarball, uri])
run([GSUTIL, 'setacl', 'public-read', uri])
finally:
if os.path.exists(tarball):
os.remove(tarball)
elif target_vm_pattern_match:
arch = target_vm_pattern_match.group(1)
mode = target_vm_pattern_match.group(2)
bot.Clobber()
tarball = 'cross_build_%s_%s.tar.bz2' % (arch, mode)
try:
test_args = [
sys.executable, test_py, '--progress=line', '--report',
'--time', '--mode=' + mode, '--arch=' + arch,
'--compiler=none', '--runtime=vm', '--write-debug-log'
]
with bot.BuildStep('Fetch build tarball'):
run([GSUTIL, 'cp', "%s/%s" % (GCS_BUCKET, tarball), tarball])
with bot.BuildStep('Unpack build tarball'):
run(['tar', '-xjf', tarball])
with bot.BuildStep('tests'):
run(test_args)
with bot.BuildStep('checked_tests'):
run(test_args + ['--checked'])
finally:
if os.path.exists(tarball):
os.remove(tarball)
else:
raise Exception("Unknown builder name %s" % name)
def SrcSteps(build_info):
# We always clobber the bot, to not leave old tarballs and packages
# floating around the out dir.
bot.Clobber(force=True)
version = utils.GetVersion()
builddir = os.path.join(bot_utils.DART_DIR, utils.GetBuildDir(HOST_OS),
'src_and_installation')
if not os.path.exists(builddir):
os.makedirs(builddir)
tarfilename = 'dart-%s.tar.gz' % version
tarfile = os.path.join(builddir, tarfilename)
with bot.BuildStep('Validating linux system'):
print 'Validating that we are on debian jessie'
args = ['cat', '/etc/os-release']
(stdout, stderr, exitcode) = bot_utils.run(args)
if exitcode != 0:
print "Could not find linux system, exiting"
sys.exit(1)
if not "jessie" in stdout:
print "Trying to build debian bits but not on debian Jessie"
print "You can't fix this, please contact dart-engprod@"
sys.exit(1)
with bot.BuildStep('Create src tarball'):
print 'Building src tarball'
Run([
sys.executable, './tools/create_tarball.py', '--tar_filename',
tarfile
])
print 'Building Debian packages'
Run([
sys.executable, './tools/create_debian_packages.py',
'--tar_filename', tarfile, '--out_dir', builddir
])
with bot.BuildStep('Sanity check installation'):
if os.path.exists('/usr/bin/dart') or os.path.exists(
'/usr/lib/dart/bin/dart2js'):
print "Dart already installed, removing"
UninstallDart()
TestInstallation(assume_installed=False)
InstallFromDep(builddir)
TestInstallation(assume_installed=True)
# We build the runtime target to get everything we need to test the
# standalone target.
Run([
sys.executable, './tools/build.py', '-mrelease', '-ax64', 'runtime'
])
# Copy in the installed binary to avoid poluting /usr/bin (and having to
# run as root)
Run(['cp', '/usr/bin/dart', 'out/ReleaseX64/dart'])
# Sanity check dart2js and the analyzer against a hello world program
with utils.TempDir() as temp_dir:
test_file = CreateDartTestFile(temp_dir)
Run(['/usr/lib/dart/bin/dart2js', test_file])
Run(['/usr/lib/dart/bin/dartanalyzer', test_file])
Run(['/usr/lib/dart/bin/dart', test_file])
# Sanity check that pub can start up and print the version
Run(['/usr/lib/dart/bin/pub', '--version'])
UninstallDart()
TestInstallation(assume_installed=False)
def Main():
name, _ = bot.GetBotName()
fletch_match = re.match(FLETCH_REGEXP, name)
cross_match = re.match(CROSS_REGEXP, name)
target_match = re.match(TARGET_REGEXP, name)
if not fletch_match and not cross_match and not target_match:
raise Exception('Invalid buildername')
SetupClangEnvironment(utils.GuessOS())
# TODO(ager/kustermann): We temporarily disable the leak detector due to
# flakiness on our buildbot of the following form:
#
# ASAN:SIGSEGV
# ==10777==LeakSanitizer has encountered a fatal error.
#
# See https://github.com/dart-lang/fletch/issues/56.
DisableMemoryLeakDetector()
# Clobber build directory if the checkbox was pressed on the BB.
with utils.ChangedWorkingDirectory(FLETCH_PATH):
bot.Clobber()
# Accumulate daemon logs messages in '.debug.log' to be displayed on the
# buildbot.Log
with open(DEBUG_LOG, 'w') as debug_log:
with utils.ChangedWorkingDirectory(FLETCH_PATH):
if fletch_match:
system = fletch_match.group(1)
modes = ['debug', 'release']
archs = ['ia32', 'x64']
asans = [False, True]
# Split configurations?
partial_configuration = fletch_match.group(2)
if partial_configuration:
mode_or_asan = fletch_match.group(3)
architecture_match = fletch_match.group(4)
archs = {
'x86' : ['ia32', 'x64'],
}[architecture_match]
# We split our builders into:
# fletch-linux-debug
# fletch-linux-release
# fletch-linux-asan (includes debug and release)
if mode_or_asan == 'asan':
modes = ['debug', 'release']
asans = [True]
else:
modes = [mode_or_asan]
asans = [False]
StepsNormal(debug_log, system, modes, archs, asans)
elif cross_match:
system = cross_match.group(1)
arch = cross_match.group(2)
assert system == 'linux'
assert arch == 'arm'
modes = ['debug', 'release']
arch = 'xarm'
StepsCrossBuilder(debug_log, system, modes, arch)
elif target_match:
assert target_match.group(1) == 'linux'
system = 'linux'
mode = target_match.group(2)
arch = 'xarm'
StepsTargetRunner(debug_log, system, mode, arch)
def SrcSteps(build_info):
# We always clobber the bot, to not leave old tarballs and packages
# floating around the out dir.
bot.Clobber(force=True)
version = utils.GetVersion()
builddir = os.path.join(bot_utils.DART_DIR, utils.GetBuildDir(HOST_OS),
'src_and_installation')
if not os.path.exists(builddir):
os.makedirs(builddir)
tarfilename = 'dart-%s.tar.gz' % version
tarfile = os.path.join(builddir, tarfilename)
with bot.BuildStep('Validating linux system'):
print 'Validating that we are on %s' % build_info.builder_tag
args = ['cat', '/etc/os-release']
(stdout, stderr, exitcode) = bot_utils.run(args)
if exitcode != 0:
print "Could not find linux system, exiting"
sys.exit(1)
if build_info.builder_tag == "debian_wheezy":
if not "jessie" in stdout:
print "Trying to build debian bits on a non debian system"
print "You can't fix this, please contact whesse@"
sys.exit(1)
if build_info.builder_tag == "ubuntu_precise":
if not "precise" in stdout:
print "Trying to build ubuntu bits on a non ubuntu system"
print "You can't fix this, please contact whesse@"
sys.exit(1)
with bot.BuildStep('Create src tarball'):
print 'Building src tarball'
Run([
sys.executable, './tools/create_tarball.py', '--tar_filename',
tarfile
])
print 'Building Debian packages'
Run([
sys.executable, './tools/create_debian_packages.py',
'--tar_filename', tarfile, '--out_dir', builddir
])
with bot.BuildStep('Sanity check installation'):
if os.path.exists('/usr/bin/dart'):
print "Dart already installled, removing"
UninstallDart()
TestInstallation(assume_installed=False)
InstallFromDep(builddir)
TestInstallation(assume_installed=True)
# We build the runtime target to get everything we need to test the
# standalone target.
Run([
sys.executable, './tools/build.py', '-mrelease', '-ax64', 'runtime'
])
# Copy in the installed binary to avoid poluting /usr/bin (and having to
# run as root)
Run(['cp', '/usr/bin/dart', 'out/ReleaseX64/dart'])
# We currently can't run the testing script on wheezy since the checked in
# binary is built on precise, see issue 18742
if (build_info.builder_tag == 'ubuntu_precise'):
Run([
sys.executable, './tools/test.py', '-ax64', '--mode=release',
'standalone'
])
# Sanity check dart2js and the analyzer against a hello world program
with utils.TempDir() as temp_dir:
test_file = CreateDartTestFile(temp_dir)
Run(['/usr/lib/dart/bin/dart2js', test_file])
Run(['/usr/lib/dart/bin/dartanalyzer', test_file])
Run(['/usr/lib/dart/bin/dart', test_file])
# Sanity check that pub can start up and print the version
Run(['/usr/lib/dart/bin/pub', '--version'])
UninstallDart()
TestInstallation(assume_installed=False)
with bot.BuildStep('Upload artifacts'):
bot_name, _ = bot.GetBotName()
channel = bot_utils.GetChannelFromName(bot_name)
if channel != bot_utils.Channel.BLEEDING_EDGE:
ArchiveArtifacts(tarfile, builddir, channel,
build_info.builder_tag)
else:
print 'Not uploading artifacts on bleeding edge'
def Main():
name, _ = bot.GetBotName()
fletch_match = re.match(FLETCH_REGEXP, name)
cross_match = re.match(CROSS_REGEXP, name)
target_match = re.match(TARGET_REGEXP, name)
if not fletch_match and not cross_match and not target_match:
raise Exception('Invalid buildername')
SetupClangEnvironment(utils.GuessOS())
SetupJavaEnvironment(utils.GuessOS())
# Clobber build directory if the checkbox was pressed on the BB.
with utils.ChangedWorkingDirectory(FLETCH_PATH):
bot.Clobber()
# Accumulate daemon logs messages in '.debug.log' to be displayed on the
# buildbot.Log
with open(DEBUG_LOG, 'w') as debug_log:
with utils.ChangedWorkingDirectory(FLETCH_PATH):
if fletch_match:
system = fletch_match.group(1)
if system == 'lk':
StepsLK(debug_log)
return
modes = ['debug', 'release']
archs = ['ia32', 'x64']
asans = [False]
# Split configurations?
partial_configuration = fletch_match.group(2)
if partial_configuration:
mode = fletch_match.group(3)
asan = fletch_match.group(4)
architecture_match = fletch_match.group(5)
archs = {
'x86': ['ia32', 'x64'],
'x64': ['x64'],
'ia32': ['ia32'],
}[architecture_match]
modes = [mode]
asans = [bool(asan)]
sdk_build = fletch_match.group(6)
if sdk_build:
StepsSDK(debug_log, system, modes, archs)
else:
StepsNormal(debug_log, system, modes, archs, asans)
elif cross_match:
system = cross_match.group(1)
arch = cross_match.group(2)
assert system == 'linux'
assert arch == 'arm'
modes = ['debug', 'release']
arch = 'xarm'
StepsCrossBuilder(debug_log, system, modes, arch)
elif target_match:
assert target_match.group(1) == 'linux'
system = 'linux'
mode = target_match.group(2)
arch = 'xarm'
StepsTargetRunner(debug_log, system, mode, arch)