def BuildScript(status, context):
inside_toolchain = context['inside_toolchain']
# Clean out build directories.
with Step('clobber', status):
RemoveSconsBuildDirectories()
RemoveGypBuildDirectories()
with Step('cleanup_temp', status):
# Picking out drive letter on which the build is happening so we can use
# it for the temp directory.
if context.Windows():
build_drive = os.path.splitdrive(os.path.abspath(__file__))[0]
tmp_dir = os.path.join(build_drive, os.path.sep + 'temp')
context.SetEnv('TEMP', tmp_dir)
context.SetEnv('TMP', tmp_dir)
else:
tmp_dir = '/tmp'
print 'Making sure %s exists...' % tmp_dir
EnsureDirectoryExists(tmp_dir)
print 'Cleaning up the contents of %s...' % tmp_dir
# Only delete files and directories like:
# */nacl_tmp/*
# TODO(bradnelson): Drop this after a bit.
# Also drop files and directories like these to cleanup current state:
# */nacl_tmp*
# */nacl*
# 83C4.tmp
# .org.chromium.Chromium.EQrEzl
# tmp_platform*
# tmp_mmap*
# tmp_pwrite*
# tmp_syscalls*
# workdir*
# nacl_chrome_download_*
# browserprofile_*
# tmp*
file_name_re = re.compile(r'[\\/\A]('
r'tmp_nacl[\\/].+|'
r'tmp_nacl.+|'
r'nacl.+|'
r'[0-9a-fA-F]+\.tmp|'
r'\.org\.chrom\w+\.Chrom\w+\.[^\\/]+|'
r'tmp_platform[^\\/]+|'
r'tmp_mmap[^\\/]+|'
r'tmp_pwrite[^\\/]+|'
r'tmp_syscalls[^\\/]+|'
r'workdir[^\\/]+|'
r'nacl_chrome_download_[^\\/]+|'
r'browserprofile_[^\\/]+|'
r'tmp[^\\/]+'
r')$')
file_name_filter = lambda fn: file_name_re.search(fn) is not None
# Clean nacl_tmp/* separately, so we get a list of leaks.
nacl_tmp = os.path.join(tmp_dir, 'nacl_tmp')
if os.path.exists(nacl_tmp):
for bot in os.listdir(nacl_tmp):
bot_path = os.path.join(nacl_tmp, bot)
print 'Cleaning prior build temp dir: %s' % bot_path
sys.stdout.flush()
if os.path.isdir(bot_path):
for d in os.listdir(bot_path):
path = os.path.join(bot_path, d)
print 'Removing leftover: %s' % path
sys.stdout.flush()
RemovePath(path)
os.rmdir(bot_path)
else:
print 'Removing rogue file: %s' % bot_path
RemovePath(bot_path)
os.rmdir(nacl_tmp)
# Clean /tmp so we get a list of what's accumulating.
TryToCleanContents(tmp_dir, file_name_filter)
# Recreate TEMP, as it may have been clobbered.
if 'TEMP' in os.environ and not os.path.exists(os.environ['TEMP']):
os.makedirs(os.environ['TEMP'])
# Mac has an additional temporary directory; clean it up.
# TODO(bradnelson): Fix Mac Chromium so that these temp files are created
# with open() + unlink() so that they will not get left behind.
if context.Mac():
subprocess.call(
"find /var/folders -name '.org.chromium.*' -exec rm -rfv '{}' ';'",
shell=True)
subprocess.call(
"find /var/folders -name '.com.google.Chrome*' -exec rm -rfv '{}' ';'",
shell=True)
# Always update Clang. On Linux and Mac, it's the default for the GN build.
# It's also used for the Linux Breakpad build. On Windows, we do a second
# Clang GN build.
with Step('update_clang', status):
Command(context,
cmd=[sys.executable, '../tools/clang/scripts/update.py'])
# Make sure our GN build is working.
using_gn = DoGNBuild(status, context)
using_gn_clang = False
if (context.Windows() and not context['clang']
and context['arch'] in ('32', '64')):
# On Windows, do a second GN build with is_clang=true.
using_gn_clang = DoGNBuild(status, context, True)
if context.Windows() and context['arch'] == '64':
# On Windows, do a second pair of GN builds for 32-bit. The 32-bit
# bots can't do GN builds at all, because the toolchains GN uses don't
# support 32-bit hosts. The 32-bit binaries built here cannot be
# tested on a 64-bit host, but compile time issues can be caught.
DoGNBuild(status, context, False, '32')
if not context['clang']:
DoGNBuild(status, context, True, '32')
# Just build both bitages of validator and test for --validator mode.
if context['validator']:
with Step('build ragel_validator-32', status):
SCons(context,
platform='x86-32',
parallel=True,
args=['ncval_new'])
with Step('build ragel_validator-64', status):
SCons(context,
platform='x86-64',
parallel=True,
args=['ncval_new'])
# Check validator trie proofs on both 32 + 64 bits.
with Step('check validator proofs', status):
SCons(context,
platform='x86-64',
parallel=False,
args=['dfachecktries'])
with Step('predownload validator corpus', status):
Command(context,
cmd=[
sys.executable,
'tests/abi_corpus/validator_regression_test.py',
'--download-only'
])
with Step('validator_regression_test ragel x86-32',
status,
halt_on_fail=False):
ValidatorTest(context, 'x86-32',
'scons-out/opt-linux-x86-32/staging/ncval_new')
with Step('validator_regression_test ragel x86-64',
status,
halt_on_fail=False):
ValidatorTest(context, 'x86-64',
'scons-out/opt-linux-x86-64/staging/ncval_new')
return
# Run checkdeps script to vet #includes.
with Step('checkdeps', status):
Command(context, cmd=[sys.executable, 'tools/checkdeps/checkdeps.py'])
# On a subset of Linux builds, build Breakpad tools for testing.
if context['use_breakpad_tools']:
with Step('breakpad configure', status):
Command(context, cmd=['mkdir', '-p', 'breakpad-out'])
# Breakpad requires C++11, so use clang and the sysroot rather than
# hoping that the host toolchain will provide support.
configure_args = []
if context.Linux():
cc = 'CC=../../third_party/llvm-build/Release+Asserts/bin/clang'
cxx = 'CXX=../../third_party/llvm-build/Release+Asserts/bin/clang++'
flags = ''
if context['arch'] == '32':
flags += ' -m32'
sysroot_arch = 'i386'
else:
flags += ' -m64'
sysroot_arch = 'amd64'
flags += (
' --sysroot=../../build/linux/debian_jessie_%s-sysroot' %
sysroot_arch)
configure_args += [cc + flags, cxx + flags]
configure_args += ['CXXFLAGS=-I../..'] # For third_party/lss
Command(context,
cwd='breakpad-out',
cmd=['bash', '../../breakpad/configure'] + configure_args)
with Step('breakpad make', status):
Command(
context,
cmd=[
'make',
'-j%d' % context['max_jobs'],
# This avoids a broken dependency on
# src/third_party/lss files within the breakpad
# source directory. We are not putting lss
# there, but using the -I switch above to
# find the lss in ../third_party instead.
'includelss_HEADERS=',
],
cwd='breakpad-out')
# The main compile step.
with Step('scons_compile', status):
SCons(context, parallel=True, args=[])
if context['coverage']:
with Step('collect_coverage', status, halt_on_fail=True):
SCons(context, args=['coverage'])
with Step('summarize_coverage', status, halt_on_fail=False):
SummarizeCoverage(context)
slave_type = os.environ.get('BUILDBOT_SLAVE_TYPE')
if slave_type != 'Trybot' and slave_type is not None:
with Step('archive_coverage', status, halt_on_fail=True):
ArchiveCoverage(context)
return
# Android bots don't run tests for now.
if context['android']:
return
### BEGIN tests ###
if not context['use_glibc']:
# Bypassing the IRT with glibc is not a supported case,
# and in fact does not work at all with the new glibc.
with Step('small_tests', status, halt_on_fail=False):
SCons(context, args=['small_tests'])
with Step('medium_tests', status, halt_on_fail=False):
SCons(context, args=['medium_tests'])
with Step('large_tests', status, halt_on_fail=False):
SCons(context, args=['large_tests'])
with Step('compile IRT tests', status):
SCons(context, parallel=True, mode=['nacl_irt_test'])
with Step('small_tests under IRT', status, halt_on_fail=False):
SCons(context,
mode=context['default_scons_mode'] + ['nacl_irt_test'],
args=['small_tests_irt'])
with Step('medium_tests under IRT', status, halt_on_fail=False):
SCons(context,
mode=context['default_scons_mode'] + ['nacl_irt_test'],
args=['medium_tests_irt'])
with Step('large_tests under IRT', status, halt_on_fail=False):
SCons(context,
mode=context['default_scons_mode'] + ['nacl_irt_test'],
args=['large_tests_irt'])
### END tests ###
### BEGIN GN tests ###
DoGNTest(status, context, using_gn, 'gn_', ' (GN)')
DoGNTest(status, context, using_gn_clang, 'gn_clang_', '(GN, Clang)')
def BuildScript(status, context):
inside_toolchain = context['inside_toolchain']
# Clean out build directories.
with Step('clobber', status):
RemoveSconsBuildDirectories()
RemoveGypBuildDirectories()
with Step('cleanup_temp', status):
# Picking out drive letter on which the build is happening so we can use
# it for the temp directory.
if context.Windows():
build_drive = os.path.splitdrive(os.path.abspath(__file__))[0]
tmp_dir = os.path.join(build_drive, os.path.sep + 'temp')
context.SetEnv('TEMP', tmp_dir)
context.SetEnv('TMP', tmp_dir)
else:
tmp_dir = '/tmp'
print 'Making sure %s exists...' % tmp_dir
EnsureDirectoryExists(tmp_dir)
print 'Cleaning up the contents of %s...' % tmp_dir
# Only delete files and directories like:
# a) C:\temp\83C4.tmp
# b) /tmp/.org.chromium.Chromium.EQrEzl
file_name_re = re.compile(
r'[\\/]([0-9a-fA-F]+\.tmp|\.org\.chrom\w+\.Chrom\w+\..+)$')
file_name_filter = lambda fn: file_name_re.search(fn) is not None
TryToCleanContents(tmp_dir, file_name_filter)
# Mac has an additional temporary directory; clean it up.
# TODO(bradnelson): Fix Mac Chromium so that these temp files are created
# with open() + unlink() so that they will not get left behind.
if context.Mac():
subprocess.call(
"find /var/folders -name '.org.chromium.*' -exec rm -rfv '{}' ';'",
shell=True)
subprocess.call(
"find /var/folders -name '.com.google.Chrome*' -exec rm -rfv '{}' ';'",
shell=True)
# Skip over hooks when run inside the toolchain build because
# download_toolchains would overwrite the toolchain build.
if inside_toolchain:
with Step('gyp_generate_only', status):
CommandGypGenerate(context)
else:
with Step('gclient_runhooks', status):
CommandGclientRunhooks(context)
if context['clang']:
with Step('update_clang', status):
Command(context, cmd=['../tools/clang/scripts/update.sh'])
# Just build both bitages of validator and test for --validator mode.
if context['validator']:
with Step('build ncval-x86-32', status):
SCons(context, platform='x86-32', parallel=True, args=['ncval'])
with Step('build ncval-x86-64', status):
SCons(context, platform='x86-64', parallel=True, args=['ncval'])
with Step('build ragel_validator-32', status):
SCons(context,
platform='x86-32',
parallel=True,
args=['ncval_new'])
with Step('build ragel_validator-64', status):
SCons(context,
platform='x86-64',
parallel=True,
args=['ncval_new'])
with Step('predownload validator corpus', status):
Command(context,
cmd=[
sys.executable,
'tests/abi_corpus/validator_regression_test.py',
'--download-only'
])
with Step('validator_regression_test current x86-32',
status,
halt_on_fail=False):
ValidatorTest(context, 'x86-32',
'scons-out/opt-linux-x86-32/staging/ncval')
with Step('validator_regression_test current x86-64',
status,
halt_on_fail=False):
ValidatorTest(context, 'x86-64',
'scons-out/opt-linux-x86-64/staging/ncval')
with Step('validator_regression_test ragel x86-32',
status,
halt_on_fail=False):
ValidatorTest(context, 'x86-32',
'scons-out/opt-linux-x86-32/staging/ncval_new')
with Step('validator_regression_test ragel x86-64',
status,
halt_on_fail=False):
ValidatorTest(context, 'x86-64',
'scons-out/opt-linux-x86-64/staging/ncval_new')
with Step('validator_diff32_tests', status, halt_on_fail=False):
SCons(context, platform='x86-32', args=['validator_diff_tests'])
with Step('validator_diff64_tests', status, halt_on_fail=False):
SCons(context, platform='x86-64', args=['validator_diff_tests'])
return
# Run checkdeps script to vet #includes.
with Step('checkdeps', status):
Command(context, cmd=[sys.executable, 'tools/checkdeps/checkdeps.py'])
# Make sure our Gyp build is working.
if not context['no_gyp']:
with Step('gyp_compile', status):
CommandGypBuild(context)
# On a subset of Linux builds, build Breakpad tools for testing.
if context['use_breakpad_tools']:
with Step('breakpad configure', status):
Command(context, cmd=['mkdir', '-p', 'breakpad-out'])
Command(
context,
cwd='breakpad-out',
cmd=['bash', '../../breakpad/configure',
'CXXFLAGS=-I../..']) # For third_party/lss
with Step('breakpad make', status):
Command(context,
cmd=['make', '-j%d' % context['max_jobs']],
cwd='breakpad-out')
# The main compile step.
with Step('scons_compile', status):
SCons(context, parallel=True, args=[])
if context['coverage']:
with Step('collect_coverage', status, halt_on_fail=True):
SCons(context, args=['coverage'])
with Step('summarize_coverage', status, halt_on_fail=False):
SummarizeCoverage(context)
slave_type = os.environ.get('BUILDBOT_SLAVE_TYPE')
if slave_type != 'Trybot' and slave_type is not None:
with Step('archive_coverage', status, halt_on_fail=True):
ArchiveCoverage(context)
return
### BEGIN tests ###
with Step('small_tests', status, halt_on_fail=False):
SCons(context, args=['small_tests'])
with Step('medium_tests', status, halt_on_fail=False):
SCons(context, args=['medium_tests'])
with Step('large_tests', status, halt_on_fail=False):
SCons(context, args=['large_tests'])
with Step('compile IRT tests', status):
SCons(context, parallel=True, mode=['nacl_irt_test'])
with Step('small_tests under IRT', status, halt_on_fail=False):
SCons(context,
mode=context['default_scons_mode'] + ['nacl_irt_test'],
args=['small_tests_irt'])
with Step('medium_tests under IRT', status, halt_on_fail=False):
SCons(context,
mode=context['default_scons_mode'] + ['nacl_irt_test'],
args=['medium_tests_irt'])
### END tests ###
if not context['no_gyp']:
# Build with ragel-based validator using GYP.
gyp_defines_save = context.GetEnv('GYP_DEFINES')
context.SetEnv('GYP_DEFINES',
' '.join([gyp_defines_save, 'nacl_validator_ragel=0']))
with Step('gyp_compile_noragel', status):
# Clobber GYP build to recompile necessary files with new preprocessor macro
# definitions. It is done because some build systems (such as GNU Make,
# MSBuild etc.) do not consider compiler arguments as a dependency.
RemoveGypBuildDirectories()
CommandGypGenerate(context)
CommandGypBuild(context)
context.SetEnv('GYP_DEFINES', gyp_defines_save)
# Build with ragel-based validator using scons.
with Step('scons_compile_noragel', status):
SCons(context, parallel=True, args=['validator_ragel=0'])
# Smoke tests for the R-DFA validator.
with Step('validator_noragel_tests', status):
args = [
'validator_ragel=0',
'small_tests',
'medium_tests',
'large_tests',
]
# Add nacl_irt_test mode to be able to run run_hello_world_test_irt that
# tests validation of the IRT.
SCons(context,
mode=context['default_scons_mode'] + ['nacl_irt_test'],
args=args)
def BuildScript(status, context):
inside_toolchain = context['inside_toolchain']
# Clean out build directories.
with Step('clobber', status):
RemoveDirectory(r'scons-out')
RemoveGypBuildDirectories()
with Step('cleanup_temp', status):
# Picking out drive letter on which the build is happening so we can use
# it for the temp directory.
if context.Windows():
build_drive = os.path.splitdrive(os.path.abspath(__file__))[0]
tmp_dir = os.path.join(build_drive, os.path.sep + 'temp')
context.SetEnv('TEMP', tmp_dir)
context.SetEnv('TMP', tmp_dir)
else:
tmp_dir = '/tmp'
print 'Making sure %s exists...' % tmp_dir
EnsureDirectoryExists(tmp_dir)
print 'Cleaning up the contents of %s...' % tmp_dir
# Only delete files and directories like:
# a) C:\temp\83C4.tmp
# b) /tmp/.org.chromium.Chromium.EQrEzl
file_name_re = re.compile(
r'[\\/]([0-9a-fA-F]+\.tmp|\.org\.chrom\w+\.Chrom\w+\..+)$')
file_name_filter = lambda fn: file_name_re.search(fn) is not None
TryToCleanContents(tmp_dir, file_name_filter)
# Mac has an additional temporary directory; clean it up.
# TODO(bradnelson): Fix Mac Chromium so that these temp files are created
# with open() + unlink() so that they will not get left behind.
if context.Mac():
subprocess.call(
"find /var/folders -name '.org.chromium.*' -exec rm -rfv '{}' ';'",
shell=True)
subprocess.call(
"find /var/folders -name '.com.google.Chrome*' -exec rm -rfv '{}' ';'",
shell=True)
# Skip over hooks when run inside the toolchain build because
# download_toolchains would overwrite the toolchain build.
if inside_toolchain:
with Step('gyp_generate_only', status):
CommandGypGenerate(context)
else:
with Step('gclient_runhooks', status):
CommandGclientRunhooks(context)
if context['clang']:
with Step('update_clang', status):
Command(context, cmd=['../tools/clang/scripts/update.sh'])
# Just build both bitages of validator and test for --validator mode.
if context['validator']:
with Step('build ncval-x86-32', status):
SCons(context, platform='x86-32', parallel=True, args=['ncval'])
with Step('build ncval-x86-64', status):
SCons(context, platform='x86-64', parallel=True, args=['ncval'])
with Step('clobber dfa_validator', status):
Command(context, cmd=['rm', '-rf', 'dfa_validator'])
with Step('clone dfa_validator', status):
Command(context, cmd=[
'git', 'clone',
'git://github.com/mseaborn/x86-decoder.git', 'dfa_validator32'])
Command(context, cmd=[
'git', 'checkout', '1a5963fa48739c586d5bbd3d46d0a8a7f25112f2'],
cwd='dfa_validator32')
Command(context, cmd=[
'git', 'clone',
'git://github.com/mseaborn/x86-decoder.git', 'dfa_validator64'])
Command(context, cmd=[
'git', 'checkout', '6ffa36f44cafd2cdad37e1e27254c498030ff712'],
cwd='dfa_validator64')
with Step('build dfa_validator_32', status):
Command(context, cmd=['make'], cwd='dfa_validator32')
with Step('build dfa_validator_64', status):
Command(context, cmd=['make'], cwd='dfa_validator64')
with Step('build ragel_validator-32', status):
SCons(context, platform='x86-32', parallel=True, args=['ncval_new'])
with Step('build ragel_validator-64', status):
SCons(context, platform='x86-64', parallel=True, args=['ncval_new'])
with Step('predownload validator corpus', status):
Command(context,
cmd=[sys.executable,
'tests/abi_corpus/validator_regression_test.py',
'--download-only'])
with Step('validator_regression_test current x86-32', status,
halt_on_fail=False):
ValidatorTest(
context, 'x86-32', 'scons-out/opt-linux-x86-32/staging/ncval')
with Step('validator_regression_test current x86-64', status,
halt_on_fail=False):
ValidatorTest(
context, 'x86-64', 'scons-out/opt-linux-x86-64/staging/ncval')
with Step('validator_regression_test dfa x86-32', status,
halt_on_fail=False):
ValidatorTest(
context, 'x86-32', 'dfa_validator32/dfa_ncval', warn_only=True)
with Step('validator_regression_test dfa x86-64', status,
halt_on_fail=False):
ValidatorTest(
context, 'x86-64', 'dfa_validator64/dfa_ncval', warn_only=True)
with Step('validator_regression_test ragel x86-32', status,
halt_on_fail=False):
ValidatorTest(
context, 'x86-32',
'scons-out/opt-linux-x86-32/staging/ncval_new')
with Step('validator_regression_test ragel x86-64', status,
halt_on_fail=False):
ValidatorTest(
context, 'x86-64',
'scons-out/opt-linux-x86-64/staging/ncval_new')
with Step('validator_diff32_tests', status, halt_on_fail=False):
SCons(context, platform='x86-32', args=['validator_diff_tests'])
with Step('validator_diff64_tests', status, halt_on_fail=False):
SCons(context, platform='x86-64', args=['validator_diff_tests'])
return
# Run checkdeps script to vet #includes.
with Step('checkdeps', status):
Command(context, cmd=[sys.executable, 'tools/checkdeps/checkdeps.py'])
# Make sure our Gyp build is working.
if not context['no_gyp']:
with Step('gyp_compile', status):
CommandGypBuild(context)
# The main compile step.
with Step('scons_compile', status):
SCons(context, parallel=True, args=[])
if context['coverage']:
with Step('collect_coverage', status, halt_on_fail=True):
SCons(context, args=['coverage'])
with Step('summarize_coverage', status, halt_on_fail=False):
SummarizeCoverage(context)
slave_type = os.environ.get('BUILDBOT_SLAVE_TYPE')
if slave_type != 'Trybot' and slave_type is not None:
with Step('archive_coverage', status, halt_on_fail=True):
ArchiveCoverage(context)
return
### BEGIN tests ###
with Step('small_tests', status, halt_on_fail=False):
SCons(context, args=['small_tests'])
with Step('medium_tests', status, halt_on_fail=False):
SCons(context, args=['medium_tests'])
with Step('large_tests', status, halt_on_fail=False):
SCons(context, args=['large_tests'])
with Step('compile IRT tests', status):
SCons(context, parallel=True, mode=['nacl_irt_test'])
with Step('small_tests under IRT', status, halt_on_fail=False):
SCons(context, mode=context['default_scons_mode'] + ['nacl_irt_test'],
args=['small_tests_irt'])
with Step('medium_tests under IRT', status, halt_on_fail=False):
SCons(context, mode=context['default_scons_mode'] + ['nacl_irt_test'],
args=['medium_tests_irt'])
# TODO(eugenis): reenable this on clang/opt once the LLVM issue is fixed
# http://code.google.com/p/nativeclient/issues/detail?id=2473
bug2473 = (context['clang'] or context['asan']) and context['mode'] == 'opt'
if context.Mac() and context['arch'] != 'arm' and not bug2473:
# We don't run all tests on x86-64 Mac yet because it would slow
# down the bots too much. We just run a small set of tests that
# have previously been fixed to pass, in order to prevent
# regressions.
# TODO(mseaborn): Remove this when we have bots dedicated to
# testing x86-64 Mac.
with Step('minimal x86-64 test', status, halt_on_fail=False):
SCons(context, parallel=True, platform='x86-64',
args=['exception_tests',
'run_hello_world_test',
'run_execute_data_test',
'run_nacl_signal_test',
'run_signal_frame_test',
'run_signal_handler_test',
'run_trusted_mmap_test'])
### END tests ###
if not context['no_gyp']:
# Build with ragel-based validator using GYP.
gyp_defines_save = context.GetEnv('GYP_DEFINES')
context.SetEnv('GYP_DEFINES',
' '.join([gyp_defines_save, 'nacl_validator_ragel=0']))
with Step('gyp_compile_ragel', status):
# Clobber GYP build to recompile necessary files with new preprocessor macro
# definitions. It is done because some build systems (such as GNU Make,
# MSBuild etc.) do not consider compiler arguments as a dependency.
RemoveGypBuildDirectories()
CommandGypGenerate(context)
CommandGypBuild(context)
context.SetEnv('GYP_DEFINES', gyp_defines_save)
# Build with ragel-based validator using scons.
with Step('scons_compile_noragel', status):
SCons(context, parallel=True, args=['validator_ragel=0'])
# Smoke tests for the R-DFA validator.
with Step('validator_noragel_tests', status):
args = ['validator_ragel=0',
'small_tests',
'medium_tests',
'large_tests',
]
# Add nacl_irt_test mode to be able to run run_hello_world_test_irt that
# tests validation of the IRT.
SCons(context,
mode=context['default_scons_mode'] + ['nacl_irt_test'],
args=args)