def main():
parser = argparse.ArgumentParser(description='Build Clang.')
parser.add_argument('--bootstrap',
action='store_true',
help='first build clang with CC, then with itself.')
parser.add_argument('--disable-asserts',
action='store_true',
help='build with asserts disabled')
parser.add_argument('--gcc-toolchain',
help='what gcc toolchain to use for '
'building; --gcc-toolchain=/opt/foo picks '
'/opt/foo/bin/gcc')
parser.add_argument('--lto-lld',
action='store_true',
help='build lld with LTO (only applies on Linux)')
parser.add_argument('--pgo', action='store_true', help='build with PGO')
parser.add_argument('--llvm-force-head-revision',
action='store_true',
help='build the latest revision')
parser.add_argument('--run-tests',
action='store_true',
help='run tests after building')
parser.add_argument('--skip-build',
action='store_true',
help='do not build anything')
parser.add_argument('--skip-checkout',
action='store_true',
help='do not create or update any checkouts')
parser.add_argument('--extra-tools',
nargs='*',
default=[],
help='select additional chrome tools to build')
parser.add_argument('--use-system-cmake',
action='store_true',
help='use the cmake from PATH instead of downloading '
'and using prebuilt cmake binaries')
parser.add_argument('--with-android',
type=gn_arg,
nargs='?',
const=True,
help='build the Android ASan runtime (linux only)',
default=sys.platform.startswith('linux'))
parser.add_argument('--without-android',
action='store_false',
help='don\'t build Android ASan runtime (linux only)',
dest='with_android')
parser.add_argument(
'--without-fuchsia',
action='store_false',
help='don\'t build Fuchsia clang_rt runtime (linux/mac)',
dest='with_fuchsia',
default=sys.platform in ('linux2', 'darwin'))
args = parser.parse_args()
if args.lto_lld and not args.bootstrap:
print('--lto-lld requires --bootstrap')
return 1
if args.lto_lld and not sys.platform.startswith('linux'):
# TODO(hans): Use it on Windows too.
print('--lto-lld is only effective on Linux. Ignoring the option.')
args.lto_lld = False
if args.pgo and not args.bootstrap:
print('--pgo requires --bootstrap')
return 1
if args.with_android and not os.path.exists(ANDROID_NDK_DIR):
print('Android NDK not found at ' + ANDROID_NDK_DIR)
print(
'The Android NDK is needed to build a Clang whose -fsanitize=address'
)
print('works on Android. See ')
print(
'https://www.chromium.org/developers/how-tos/android-build-instructions'
)
print('for how to install the NDK, or pass --without-android.')
return 1
if args.llvm_force_head_revision:
# Don't build fuchsia runtime on ToT bots at all.
args.with_fuchsia = False
if args.with_fuchsia and not os.path.exists(FUCHSIA_SDK_DIR):
print('Fuchsia SDK not found at ' + FUCHSIA_SDK_DIR)
print('The Fuchsia SDK is needed to build libclang_rt for Fuchsia.')
print('Install the Fuchsia SDK by adding fuchsia to the ')
print('target_os section in your .gclient and running hooks, ')
print('or pass --without-fuchsia.')
print(
'https://chromium.googlesource.com/chromium/src/+/master/docs/fuchsia_build_instructions.md'
)
print('for general Fuchsia build instructions.')
return 1
# Don't buffer stdout, so that print statements are immediately flushed.
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
# The gnuwin package also includes curl, which is needed to interact with the
# github API below.
# TODO(crbug.com/965937): Use urllib once our Python is recent enough, and
# move this down to where we fetch other build tools.
AddGnuWinToPath()
# TODO(crbug.com/929645): Remove once we build on host systems with a modern
# enough GCC to build Clang.
MaybeDownloadHostGcc(args)
global CLANG_REVISION, PACKAGE_VERSION
if args.llvm_force_head_revision:
CLANG_REVISION = GetLatestLLVMCommit()
if not args.skip_checkout:
CheckoutLLVM(CLANG_REVISION, LLVM_DIR)
if args.llvm_force_head_revision:
PACKAGE_VERSION = 'n%s-%s-0' % (GetCommitCount(CLANG_REVISION),
CLANG_REVISION[:8])
print('Locally building clang %s...' % PACKAGE_VERSION)
WriteStampFile('', STAMP_FILE)
WriteStampFile('', FORCE_HEAD_REVISION_FILE)
AddCMakeToPath(args)
DeleteChromeToolsShim()
if args.skip_build:
return 0
# The variable "lld" is only used on Windows because only there does setting
# CMAKE_LINKER have an effect: On Windows, the linker is called directly,
# while elsewhere it's called through the compiler driver, and we pass
# -fuse-ld=lld there to make the compiler driver call the linker (by setting
# LLVM_ENABLE_LLD).
cc, cxx, lld = None, None, None
if args.gcc_toolchain:
# Use the specified gcc installation for building.
cc = os.path.join(args.gcc_toolchain, 'bin', 'gcc')
cxx = os.path.join(args.gcc_toolchain, 'bin', 'g++')
if not os.access(cc, os.X_OK):
print('Invalid --gcc-toolchain: ' + args.gcc_toolchain)
return 1
cflags = []
cxxflags = []
ldflags = []
targets = 'AArch64;ARM;Mips;PowerPC;SystemZ;WebAssembly;X86'
projects = 'clang;compiler-rt;lld;chrometools'
if sys.platform == 'darwin':
# clang needs libc++, else -stdlib=libc++ won't find includes
# (this is needed for bootstrap builds and for building the fuchsia runtime)
projects += ';libcxx'
base_cmake_args = [
'-GNinja',
'-DCMAKE_BUILD_TYPE=Release',
'-DLLVM_ENABLE_ASSERTIONS=%s' %
('OFF' if args.disable_asserts else 'ON'),
'-DLLVM_ENABLE_PROJECTS=' + projects,
'-DLLVM_TARGETS_TO_BUILD=' + targets,
'-DLLVM_ENABLE_PIC=OFF',
'-DLLVM_ENABLE_UNWIND_TABLES=OFF',
'-DLLVM_ENABLE_TERMINFO=OFF',
'-DCLANG_PLUGIN_SUPPORT=OFF',
'-DCLANG_ENABLE_STATIC_ANALYZER=OFF',
'-DCLANG_ENABLE_ARCMT=OFF',
'-DBUG_REPORT_URL=' + BUG_REPORT_URL,
# See PR41956: Don't link libcxx into libfuzzer.
'-DCOMPILER_RT_USE_LIBCXX=NO',
# Don't run Go bindings tests; PGO makes them confused.
'-DLLVM_INCLUDE_GO_TESTS=OFF',
]
if args.gcc_toolchain:
# Don't use the custom gcc toolchain when building compiler-rt tests; those
# tests are built with the just-built Clang, and target both i386 and x86_64
# for example, so should ust the system's libstdc++.
base_cmake_args.append(
'-DCOMPILER_RT_TEST_COMPILER_CFLAGS=--gcc-toolchain=')
if sys.platform == 'win32':
base_cmake_args.append('-DLLVM_USE_CRT_RELEASE=MT')
if sys.platform == 'darwin':
# Use the system libc++abi.
# TODO(hans): use https://reviews.llvm.org/D62060 instead
base_cmake_args.append('-DLIBCXX_CXX_ABI=libcxxabi')
base_cmake_args.append('-DLIBCXX_CXX_ABI_SYSTEM=1')
if sys.platform != 'win32':
# libxml2 is required by the Win manifest merging tool used in cross-builds.
base_cmake_args.append('-DLLVM_ENABLE_LIBXML2=FORCE_ON')
if args.bootstrap:
print('Building bootstrap compiler')
if os.path.exists(LLVM_BOOTSTRAP_DIR):
RmTree(LLVM_BOOTSTRAP_DIR)
EnsureDirExists(LLVM_BOOTSTRAP_DIR)
os.chdir(LLVM_BOOTSTRAP_DIR)
projects = 'clang'
if args.pgo:
# Need libclang_rt.profile
projects += ';compiler-rt'
if sys.platform != 'darwin' or args.lto_lld:
projects += ';lld'
if sys.platform == 'darwin':
# Need libc++ and compiler-rt for the bootstrap compiler on mac.
projects += ';libcxx;compiler-rt'
bootstrap_targets = 'X86'
if sys.platform == 'darwin':
# Need ARM and AArch64 for building the ios clang_rt.
bootstrap_targets += ';ARM;AArch64'
bootstrap_args = base_cmake_args + [
'-DLLVM_TARGETS_TO_BUILD=' + bootstrap_targets,
'-DLLVM_ENABLE_PROJECTS=' + projects,
'-DCMAKE_INSTALL_PREFIX=' + LLVM_BOOTSTRAP_INSTALL_DIR,
'-DCMAKE_C_FLAGS=' + ' '.join(cflags),
'-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
# Ignore args.disable_asserts for the bootstrap compiler.
'-DLLVM_ENABLE_ASSERTIONS=ON',
]
if sys.platform == 'darwin':
# On macOS, the bootstrap toolchain needs to have compiler-rt because
# dsymutil's link needs libclang_rt.osx.a. Only the x86_64 osx
# libraries are needed though, and only libclang_rt (i.e.
# COMPILER_RT_BUILD_BUILTINS).
bootstrap_args.extend([
'-DDARWIN_osx_ARCHS=x86_64',
'-DCOMPILER_RT_BUILD_BUILTINS=ON',
'-DCOMPILER_RT_BUILD_CRT=OFF',
'-DCOMPILER_RT_BUILD_LIBFUZZER=OFF',
'-DCOMPILER_RT_BUILD_SANITIZERS=OFF',
'-DCOMPILER_RT_BUILD_XRAY=OFF',
'-DCOMPILER_RT_ENABLE_IOS=OFF',
'-DCOMPILER_RT_ENABLE_WATCHOS=OFF',
'-DCOMPILER_RT_ENABLE_TVOS=OFF',
])
elif args.pgo:
# PGO needs libclang_rt.profile but none of the other compiler-rt stuff.
bootstrap_args.extend([
'-DCOMPILER_RT_BUILD_BUILTINS=OFF',
'-DCOMPILER_RT_BUILD_CRT=OFF',
'-DCOMPILER_RT_BUILD_LIBFUZZER=OFF',
'-DCOMPILER_RT_BUILD_PROFILE=ON',
'-DCOMPILER_RT_BUILD_SANITIZERS=OFF',
'-DCOMPILER_RT_BUILD_XRAY=OFF',
])
if cc is not None: bootstrap_args.append('-DCMAKE_C_COMPILER=' + cc)
if cxx is not None:
bootstrap_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
if lld is not None: bootstrap_args.append('-DCMAKE_LINKER=' + lld)
RunCommand(['cmake'] + bootstrap_args +
[os.path.join(LLVM_DIR, 'llvm')],
msvc_arch='x64')
CopyLibstdcpp(args, LLVM_BOOTSTRAP_DIR)
CopyLibstdcpp(args, LLVM_BOOTSTRAP_INSTALL_DIR)
RunCommand(['ninja'], msvc_arch='x64')
if args.run_tests:
test_targets = ['check-all']
if sys.platform == 'darwin':
# TODO(crbug.com/731375): Run check-all on Darwin too.
test_targets = ['check-llvm', 'check-clang', 'check-builtins']
if sys.platform == 'win32':
CopyDiaDllTo(os.path.join(LLVM_BOOTSTRAP_DIR, 'bin'))
RunCommand(['ninja'] + test_targets, msvc_arch='x64')
RunCommand(['ninja', 'install'], msvc_arch='x64')
if sys.platform == 'win32':
cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin',
'clang-cl.exe')
cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin',
'clang-cl.exe')
lld = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin',
'lld-link.exe')
# CMake has a hard time with backslashes in compiler paths:
# https://stackoverflow.com/questions/13050827
cc = cc.replace('\\', '/')
cxx = cxx.replace('\\', '/')
lld = lld.replace('\\', '/')
else:
cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang')
cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang++')
if sys.platform.startswith('linux'):
base_cmake_args.append('-DLLVM_ENABLE_LLD=ON')
if args.gcc_toolchain:
# Tell the bootstrap compiler where to find the standard library headers
# and shared object files.
cflags.append('--gcc-toolchain=' + args.gcc_toolchain)
cxxflags.append('--gcc-toolchain=' + args.gcc_toolchain)
print('Bootstrap compiler installed.')
if args.pgo:
print('Building instrumented compiler')
if os.path.exists(LLVM_INSTRUMENTED_DIR):
RmTree(LLVM_INSTRUMENTED_DIR)
EnsureDirExists(LLVM_INSTRUMENTED_DIR)
os.chdir(LLVM_INSTRUMENTED_DIR)
projects = 'clang'
if sys.platform == 'darwin':
projects += ';libcxx;compiler-rt'
instrument_args = base_cmake_args + [
'-DLLVM_ENABLE_THREADS=OFF',
'-DLLVM_ENABLE_PROJECTS=' + projects,
'-DCMAKE_C_FLAGS=' + ' '.join(cflags),
'-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
# Build with instrumentation.
'-DLLVM_BUILD_INSTRUMENTED=IR',
]
# Build with the bootstrap compiler.
if cc is not None: instrument_args.append('-DCMAKE_C_COMPILER=' + cc)
if cxx is not None:
instrument_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
if lld is not None: instrument_args.append('-DCMAKE_LINKER=' + lld)
RunCommand(['cmake'] + instrument_args +
[os.path.join(LLVM_DIR, 'llvm')],
msvc_arch='x64')
CopyLibstdcpp(args, LLVM_INSTRUMENTED_DIR)
RunCommand(['ninja'], msvc_arch='x64')
print('Instrumented compiler built.')
# Train by building some C++ code.
#
# pgo_training-1.ii is a preprocessed (on Linux) version of
# src/third_party/blink/renderer/core/layout/layout_object.cc, selected
# because it's a large translation unit in Blink, which is normally the
# slowest part of Chromium to compile. Using this, we get ~20% shorter
# build times for Linux, Android, and Mac, which is also what we got when
# training by actually building a target in Chromium. (For comparison, a
# C++-y "Hello World" program only resulted in 14% faster builds.)
# See https://crbug.com/966403#c16 for all numbers.
#
# TODO(hans): Enhance the training, perhaps by including preprocessed code
# from more platforms, and by doing some linking so that lld can benefit
# from PGO as well. Perhaps the training could be done asynchronously by
# dedicated buildbots that upload profiles to the cloud.
training_source = 'pgo_training-1.ii'
with open(training_source, 'w') as f:
DownloadUrl(CDS_URL + '/' + training_source, f)
train_cmd = [
os.path.join(LLVM_INSTRUMENTED_DIR, 'bin', 'clang++'), '-target',
'x86_64-unknown-unknown', '-O2', '-g', '-std=c++14',
'-fno-exceptions', '-fno-rtti', '-w', '-c', training_source
]
if sys.platform == 'darwin':
train_cmd.extend([
'-stdlib=libc++', '-isysroot',
subprocess.check_output(['xcrun', '--show-sdk-path']).rstrip()
])
RunCommand(train_cmd, msvc_arch='x64')
# Merge profiles.
profdata = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin',
'llvm-profdata')
RunCommand(
[profdata, 'merge', '-output=' + LLVM_PROFDATA_FILE] + glob.glob(
os.path.join(LLVM_INSTRUMENTED_DIR, 'profiles', '*.profraw')),
msvc_arch='x64')
print('Profile generated.')
compiler_rt_args = [
'-DCOMPILER_RT_BUILD_CRT=OFF',
'-DCOMPILER_RT_BUILD_LIBFUZZER=OFF',
'-DCOMPILER_RT_BUILD_PROFILE=ON',
'-DCOMPILER_RT_BUILD_SANITIZERS=ON',
'-DCOMPILER_RT_BUILD_XRAY=OFF',
]
if sys.platform == 'darwin':
compiler_rt_args.extend([
'-DCOMPILER_RT_BUILD_BUILTINS=ON',
'-DCOMPILER_RT_ENABLE_IOS=ON',
'-DCOMPILER_RT_ENABLE_WATCHOS=OFF',
'-DCOMPILER_RT_ENABLE_TVOS=OFF',
# armv7 is A5 and earlier, armv7s is A6+ (2012 and later, before 64-bit
# iPhones). armv7k is Apple Watch, which we don't need.
'-DDARWIN_ios_ARCHS=armv7;armv7s;arm64',
'-DDARWIN_iossim_ARCHS=i386;x86_64',
# We don't need 32-bit intel support for macOS, we only ship 64-bit.
'-DDARWIN_osx_ARCHS=x86_64',
])
else:
compiler_rt_args.append('-DCOMPILER_RT_BUILD_BUILTINS=OFF')
# LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is
# needed, on OS X it requires libc++. clang only automatically links to libc++
# when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run
# on OS X versions as old as 10.7.
deployment_target = ''
if sys.platform == 'darwin' and args.bootstrap:
# When building on 10.9, /usr/include usually doesn't exist, and while
# Xcode's clang automatically sets a sysroot, self-built clangs don't.
cflags = [
'-isysroot',
subprocess.check_output(['xcrun', '--show-sdk-path']).rstrip()
]
cxxflags = ['-stdlib=libc++'] + cflags
ldflags += ['-stdlib=libc++']
deployment_target = '10.7'
# If building at head, define a macro that plugins can use for #ifdefing
# out code that builds at head, but not at CLANG_REVISION or vice versa.
if args.llvm_force_head_revision:
cflags += ['-DLLVM_FORCE_HEAD_REVISION']
cxxflags += ['-DLLVM_FORCE_HEAD_REVISION']
# Build PDBs for archival on Windows. Don't use RelWithDebInfo since it
# has different optimization defaults than Release.
# Also disable stack cookies (/GS-) for performance.
if sys.platform == 'win32':
cflags += ['/Zi', '/GS-']
cxxflags += ['/Zi', '/GS-']
ldflags += ['/DEBUG', '/OPT:REF', '/OPT:ICF']
deployment_env = None
if deployment_target:
deployment_env = os.environ.copy()
deployment_env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target
# Build lld and code coverage tools. This is done separately from the rest of
# the build because these tools require threading support.
print('Building thread-enabled tools.')
tools_with_threading = ['dsymutil', 'lld', 'llvm-cov', 'llvm-profdata']
print('Building the following tools with threading support: %s' %
str(tools_with_threading))
if os.path.exists(THREADS_ENABLED_BUILD_DIR):
RmTree(THREADS_ENABLED_BUILD_DIR)
EnsureDirExists(THREADS_ENABLED_BUILD_DIR)
os.chdir(THREADS_ENABLED_BUILD_DIR)
threads_enabled_cmake_args = base_cmake_args + [
'-DCMAKE_C_FLAGS=' + ' '.join(cflags), '-DCMAKE_CXX_FLAGS=' +
' '.join(cxxflags), '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
'-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
'-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags)
]
if cc is not None:
threads_enabled_cmake_args.append('-DCMAKE_C_COMPILER=' + cc)
if cxx is not None:
threads_enabled_cmake_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
if lld is not None:
threads_enabled_cmake_args.append('-DCMAKE_LINKER=' + lld)
if args.lto_lld:
# Build lld with LTO. That speeds up the linker by ~10%.
# We only use LTO for Linux now.
#
# The linker expects all archive members to have symbol tables, so the
# archiver needs to be able to create symbol tables for bitcode files.
# GNU ar and ranlib don't understand bitcode files, but llvm-ar and
# llvm-ranlib do, so use them.
ar = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'llvm-ar')
ranlib = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'llvm-ranlib')
threads_enabled_cmake_args += [
'-DCMAKE_AR=' + ar,
'-DCMAKE_RANLIB=' + ranlib,
'-DLLVM_ENABLE_LTO=thin',
]
RunCommand(['cmake'] + threads_enabled_cmake_args +
[os.path.join(LLVM_DIR, 'llvm')],
msvc_arch='x64',
env=deployment_env)
CopyLibstdcpp(args, THREADS_ENABLED_BUILD_DIR)
RunCommand(['ninja'] + tools_with_threading, msvc_arch='x64')
print('Building final compiler.')
default_tools = ['plugins', 'blink_gc_plugin', 'translation_unit']
chrome_tools = list(set(default_tools + args.extra_tools))
if cc is not None: base_cmake_args.append('-DCMAKE_C_COMPILER=' + cc)
if cxx is not None: base_cmake_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
if lld is not None: base_cmake_args.append('-DCMAKE_LINKER=' + lld)
cmake_args = base_cmake_args + compiler_rt_args + [
'-DLLVM_ENABLE_THREADS=OFF', '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
'-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
'-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
'-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
'-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
'-DCMAKE_INSTALL_PREFIX=' + LLVM_BUILD_DIR,
'-DCHROMIUM_TOOLS_SRC=%s' %
os.path.join(CHROMIUM_DIR, 'tools', 'clang'),
'-DCHROMIUM_TOOLS=%s' % ';'.join(chrome_tools)
]
if args.pgo:
cmake_args.append('-DLLVM_PROFDATA_FILE=' + LLVM_PROFDATA_FILE)
if sys.platform == 'darwin':
cmake_args += [
'-DCOMPILER_RT_ENABLE_IOS=ON', '-DSANITIZER_MIN_OSX_VERSION=10.7'
]
# TODO(crbug.com/962988): Use -DLLVM_EXTERNAL_PROJECTS instead.
CreateChromeToolsShim()
if os.path.exists(LLVM_BUILD_DIR):
RmTree(LLVM_BUILD_DIR)
EnsureDirExists(LLVM_BUILD_DIR)
os.chdir(LLVM_BUILD_DIR)
RunCommand(['cmake'] + cmake_args + [os.path.join(LLVM_DIR, 'llvm')],
msvc_arch='x64',
env=deployment_env)
CopyLibstdcpp(args, LLVM_BUILD_DIR)
RunCommand(['ninja'], msvc_arch='x64')
# Copy in the threaded versions of lld and other tools.
if sys.platform == 'win32':
CopyFile(
os.path.join(THREADS_ENABLED_BUILD_DIR, 'bin', 'lld-link.exe'),
os.path.join(LLVM_BUILD_DIR, 'bin'))
CopyFile(os.path.join(THREADS_ENABLED_BUILD_DIR, 'bin', 'lld.pdb'),
os.path.join(LLVM_BUILD_DIR, 'bin'))
else:
for tool in tools_with_threading:
CopyFile(os.path.join(THREADS_ENABLED_BUILD_DIR, 'bin', tool),
os.path.join(LLVM_BUILD_DIR, 'bin'))
if chrome_tools:
# If any Chromium tools were built, install those now.
RunCommand(['ninja', 'cr-install'], msvc_arch='x64')
VerifyVersionOfBuiltClangMatchesVERSION()
if sys.platform == 'win32':
platform = 'windows'
elif sys.platform == 'darwin':
platform = 'darwin'
else:
assert sys.platform.startswith('linux')
platform = 'linux'
rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
RELEASE_VERSION, 'lib', platform)
# Do an out-of-tree build of compiler-rt for 32-bit Win clang_rt.profile.lib.
if sys.platform == 'win32':
if os.path.isdir(COMPILER_RT_BUILD_DIR):
RmTree(COMPILER_RT_BUILD_DIR)
os.makedirs(COMPILER_RT_BUILD_DIR)
os.chdir(COMPILER_RT_BUILD_DIR)
if args.bootstrap:
# The bootstrap compiler produces 64-bit binaries by default.
cflags += ['-m32']
cxxflags += ['-m32']
compiler_rt_args = base_cmake_args + [
'-DLLVM_ENABLE_THREADS=OFF',
'-DCMAKE_C_FLAGS=' + ' '.join(cflags),
'-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
'-DCOMPILER_RT_BUILD_BUILTINS=OFF',
'-DCOMPILER_RT_BUILD_CRT=OFF',
'-DCOMPILER_RT_BUILD_LIBFUZZER=OFF',
'-DCOMPILER_RT_BUILD_PROFILE=ON',
'-DCOMPILER_RT_BUILD_SANITIZERS=OFF',
'-DCOMPILER_RT_BUILD_XRAY=OFF',
]
RunCommand(['cmake'] + compiler_rt_args +
[os.path.join(LLVM_DIR, 'llvm')],
msvc_arch='x86',
env=deployment_env)
RunCommand(['ninja', 'compiler-rt'], msvc_arch='x86')
# Copy select output to the main tree.
rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang',
RELEASE_VERSION, 'lib', platform)
# Static and dynamic libraries:
CopyDirectoryContents(rt_lib_src_dir, rt_lib_dst_dir)
if args.with_android:
make_toolchain = os.path.join(ANDROID_NDK_DIR, 'build', 'tools',
'make_standalone_toolchain.py')
# TODO(thakis): Now that the NDK uses clang, try to build all archs in
# one LLVM build instead of making 3 different toolchains and building
# 3 times.
for target_arch in ['aarch64', 'arm', 'i686']:
# Make standalone Android toolchain for target_arch.
toolchain_dir = os.path.join(LLVM_BUILD_DIR,
'android-toolchain-' + target_arch)
api_level = '21' if target_arch == 'aarch64' else '19'
RunCommand([
make_toolchain, '--api=' + api_level, '--force',
'--install-dir=%s' % toolchain_dir, '--stl=libc++',
'--arch=' + {
'aarch64': 'arm64',
'arm': 'arm',
'i686': 'x86',
}[target_arch]
])
# Build compiler-rt runtimes needed for Android in a separate build tree.
build_dir = os.path.join(LLVM_BUILD_DIR, 'android-' + target_arch)
if not os.path.exists(build_dir):
os.mkdir(os.path.join(build_dir))
os.chdir(build_dir)
target_triple = target_arch
if target_arch == 'arm':
target_triple = 'armv7'
target_triple += '-linux-android' + api_level
cflags = [
'--target=%s' % target_triple,
'--sysroot=%s/sysroot' % toolchain_dir,
'-B%s' % toolchain_dir
]
android_args = base_cmake_args + [
'-DLLVM_ENABLE_THREADS=OFF', '-DCMAKE_C_COMPILER=' +
os.path.join(LLVM_BUILD_DIR, 'bin/clang'),
'-DCMAKE_CXX_COMPILER=' + os.path.join(
LLVM_BUILD_DIR, 'bin/clang++'), '-DLLVM_CONFIG_PATH=' +
os.path.join(LLVM_BUILD_DIR, 'bin/llvm-config'),
'-DCMAKE_C_FLAGS=' + ' '.join(cflags), '-DCMAKE_CXX_FLAGS=' +
' '.join(cflags), '-DCMAKE_ASM_FLAGS=' + ' '.join(cflags),
'-DCOMPILER_RT_BUILD_BUILTINS=OFF',
'-DCOMPILER_RT_BUILD_CRT=OFF',
'-DCOMPILER_RT_BUILD_LIBFUZZER=OFF',
'-DCOMPILER_RT_BUILD_PROFILE=ON',
'-DCOMPILER_RT_BUILD_SANITIZERS=ON',
'-DCOMPILER_RT_BUILD_XRAY=OFF',
'-DSANITIZER_CXX_ABI=libcxxabi',
'-DCMAKE_SHARED_LINKER_FLAGS=-Wl,-u__cxa_demangle',
'-DANDROID=1'
]
RunCommand(['cmake'] + android_args + [COMPILER_RT_DIR])
# We use ASan i686 build for fuzzing.
libs_want = ['lib/linux/libclang_rt.asan-{0}-android.so']
if target_arch in ['aarch64', 'arm']:
libs_want += [
'lib/linux/libclang_rt.ubsan_standalone-{0}-android.so',
'lib/linux/libclang_rt.profile-{0}-android.a',
]
if target_arch == 'aarch64':
libs_want += ['lib/linux/libclang_rt.hwasan-{0}-android.so']
libs_want = [lib.format(target_arch) for lib in libs_want]
RunCommand(['ninja'] + libs_want)
# And copy them into the main build tree.
for p in libs_want:
shutil.copy(p, rt_lib_dst_dir)
if args.with_fuchsia:
# Fuchsia links against libclang_rt.builtins-<arch>.a instead of libgcc.a.
for target_arch in ['aarch64', 'x86_64']:
fuchsia_arch_name = {
'aarch64': 'arm64',
'x86_64': 'x64'
}[target_arch]
toolchain_dir = os.path.join(FUCHSIA_SDK_DIR, 'arch',
fuchsia_arch_name, 'sysroot')
# Build clang_rt runtime for Fuchsia in a separate build tree.
build_dir = os.path.join(LLVM_BUILD_DIR, 'fuchsia-' + target_arch)
if not os.path.exists(build_dir):
os.mkdir(os.path.join(build_dir))
os.chdir(build_dir)
target_spec = target_arch + '-fuchsia'
# TODO(thakis): Might have to pass -B here once sysroot contains
# binaries (e.g. gas for arm64?)
fuchsia_args = base_cmake_args + [
'-DLLVM_ENABLE_THREADS=OFF',
'-DCMAKE_C_COMPILER=' +
os.path.join(LLVM_BUILD_DIR, 'bin/clang'),
'-DCMAKE_CXX_COMPILER=' +
os.path.join(LLVM_BUILD_DIR, 'bin/clang++'),
'-DCMAKE_LINKER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang'),
'-DCMAKE_AR=' + os.path.join(LLVM_BUILD_DIR, 'bin/llvm-ar'),
'-DLLVM_CONFIG_PATH=' +
os.path.join(LLVM_BUILD_DIR, 'bin/llvm-config'),
'-DCMAKE_SYSTEM_NAME=Fuchsia',
'-DCMAKE_C_COMPILER_TARGET=%s-fuchsia' % target_arch,
'-DCMAKE_ASM_COMPILER_TARGET=%s-fuchsia' % target_arch,
'-DCOMPILER_RT_BUILD_BUILTINS=ON',
'-DCOMPILER_RT_BUILD_CRT=OFF',
'-DCOMPILER_RT_BUILD_LIBFUZZER=OFF',
'-DCOMPILER_RT_BUILD_PROFILE=OFF',
'-DCOMPILER_RT_BUILD_SANITIZERS=OFF',
'-DCOMPILER_RT_BUILD_XRAY=OFF',
'-DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON',
'-DCMAKE_SYSROOT=%s' % toolchain_dir,
# TODO(thakis|scottmg): Use PER_TARGET_RUNTIME_DIR for all platforms.
# https://crbug.com/882485.
'-DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=ON',
# These are necessary because otherwise CMake tries to build an
# executable to test to see if the compiler is working, but in doing so,
# it links against the builtins.a that we're about to build.
'-DCMAKE_C_COMPILER_WORKS=ON',
'-DCMAKE_ASM_COMPILER_WORKS=ON',
]
RunCommand(['cmake'] + fuchsia_args +
[os.path.join(COMPILER_RT_DIR, 'lib', 'builtins')])
builtins_a = 'libclang_rt.builtins.a'
RunCommand(['ninja', builtins_a])
# And copy it into the main build tree.
fuchsia_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
RELEASE_VERSION, 'lib',
target_spec)
if not os.path.exists(fuchsia_lib_dst_dir):
os.makedirs(fuchsia_lib_dst_dir)
CopyFile(os.path.join(build_dir, 'lib', target_spec, builtins_a),
fuchsia_lib_dst_dir)
# Run tests.
if args.run_tests or args.llvm_force_head_revision:
RunCommand(['ninja', '-C', LLVM_BUILD_DIR, 'cr-check-all'],
msvc_arch='x64')
if args.run_tests:
if sys.platform == 'win32':
CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin'))
test_targets = ['check-all']
if sys.platform == 'darwin':
# TODO(thakis): Run check-all on Darwin too, https://crbug.com/959361
test_targets = ['check-llvm', 'check-clang', 'check-lld']
RunCommand(['ninja', '-C', LLVM_BUILD_DIR] + test_targets,
msvc_arch='x64')
if sys.platform == 'darwin':
for dylib in glob.glob(os.path.join(rt_lib_dst_dir, '*.dylib')):
# Fix LC_ID_DYLIB for the ASan dynamic libraries to be relative to
# @executable_path.
# Has to happen after running tests.
# TODO(glider): this is transitional. We'll need to fix the dylib
# name either in our build system, or in Clang. See also
# http://crbug.com/344836.
subprocess.call([
'install_name_tool', '-id',
'@executable_path/' + os.path.basename(dylib), dylib
])
WriteStampFile(PACKAGE_VERSION, STAMP_FILE)
WriteStampFile(PACKAGE_VERSION, FORCE_HEAD_REVISION_FILE)
print('Clang build was successful.')
return 0
def main():
parser = argparse.ArgumentParser(description='Build Clang.')
parser.add_argument('--bootstrap', action='store_true',
help='first build clang with CC, then with itself.')
parser.add_argument('--disable-asserts', action='store_true',
help='build with asserts disabled')
parser.add_argument('--gcc-toolchain', help='what gcc toolchain to use for '
'building; --gcc-toolchain=/opt/foo picks '
'/opt/foo/bin/gcc')
parser.add_argument('--pgo', action='store_true', help='build with PGO')
parser.add_argument('--thinlto',
action='store_true',
help='build with ThinLTO')
parser.add_argument('--llvm-force-head-revision', action='store_true',
help='build the latest revision')
parser.add_argument('--run-tests', action='store_true',
help='run tests after building')
parser.add_argument('--skip-build', action='store_true',
help='do not build anything')
parser.add_argument('--skip-checkout', action='store_true',
help='do not create or update any checkouts')
parser.add_argument('--build-dir',
help='Override build directory')
parser.add_argument('--extra-tools', nargs='*', default=[],
help='select additional chrome tools to build')
parser.add_argument('--use-system-cmake', action='store_true',
help='use the cmake from PATH instead of downloading '
'and using prebuilt cmake binaries')
parser.add_argument('--with-android', type=gn_arg, nargs='?', const=True,
help='build the Android ASan runtime (linux only)',
default=sys.platform.startswith('linux'))
parser.add_argument('--with-fuchsia',
type=gn_arg,
nargs='?',
const=True,
help='build the Fuchsia runtimes (linux and mac only)',
default=sys.platform.startswith('linux')
or sys.platform.startswith('darwin'))
parser.add_argument('--without-android', action='store_false',
help='don\'t build Android ASan runtime (linux only)',
dest='with_android')
parser.add_argument('--without-fuchsia', action='store_false',
help='don\'t build Fuchsia clang_rt runtime (linux/mac)',
dest='with_fuchsia',
default=sys.platform in ('linux2', 'darwin'))
args = parser.parse_args()
if (args.pgo or args.thinlto) and not args.bootstrap:
print('--pgo/--thinlto requires --bootstrap')
return 1
if args.with_android and not os.path.exists(ANDROID_NDK_DIR):
print('Android NDK not found at ' + ANDROID_NDK_DIR)
print('The Android NDK is needed to build a Clang whose -fsanitize=address')
print('works on Android. See ')
print('https://www.chromium.org/developers/how-tos/android-build-instructions')
print('for how to install the NDK, or pass --without-android.')
return 1
if args.with_fuchsia and not os.path.exists(FUCHSIA_SDK_DIR):
print('Fuchsia SDK not found at ' + FUCHSIA_SDK_DIR)
print('The Fuchsia SDK is needed to build libclang_rt for Fuchsia.')
print('Install the Fuchsia SDK by adding fuchsia to the ')
print('target_os section in your .gclient and running hooks, ')
print('or pass --without-fuchsia.')
print(
'https://chromium.googlesource.com/chromium/src/+/master/docs/fuchsia/build_instructions.md'
)
print('for general Fuchsia build instructions.')
return 1
# Don't buffer stdout, so that print statements are immediately flushed.
# LLVM tests print output without newlines, so with buffering they won't be
# immediately printed.
major, _, _, _, _ = sys.version_info
if major == 3:
# Python3 only allows unbuffered output for binary streams. This
# workaround comes from https://stackoverflow.com/a/181654/4052492.
sys.stdout = io.TextIOWrapper(open(sys.stdout.fileno(), 'wb', 0),
write_through=True)
else:
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
# The gnuwin package also includes curl, which is needed to interact with the
# github API below.
# TODO(crbug.com/1067752): Use urllib once certificates are fixed, and
# move this down to where we fetch other build tools.
AddGnuWinToPath()
# TODO(crbug.com/929645): Remove once we build on host systems with a modern
# enough GCC to build Clang.
MaybeDownloadHostGcc(args)
if sys.platform == 'darwin':
isysroot = subprocess.check_output(['xcrun', '--show-sdk-path']).rstrip()
global CLANG_REVISION, PACKAGE_VERSION, LLVM_BUILD_DIR
if args.build_dir:
LLVM_BUILD_DIR = args.build_dir
if args.llvm_force_head_revision:
checkout_revision = GetLatestLLVMCommit()
else:
checkout_revision = CLANG_REVISION
if not args.skip_checkout:
CheckoutLLVM(checkout_revision, LLVM_DIR)
if args.llvm_force_head_revision:
CLANG_REVISION = GetCommitDescription(checkout_revision)
PACKAGE_VERSION = '%s-0' % CLANG_REVISION
print('Locally building clang %s...' % PACKAGE_VERSION)
WriteStampFile('', STAMP_FILE)
WriteStampFile('', FORCE_HEAD_REVISION_FILE)
AddCMakeToPath(args)
DeleteChromeToolsShim()
if args.skip_build:
return 0
# The variable "lld" is only used on Windows because only there does setting
# CMAKE_LINKER have an effect: On Windows, the linker is called directly,
# while elsewhere it's called through the compiler driver, and we pass
# -fuse-ld=lld there to make the compiler driver call the linker (by setting
# LLVM_ENABLE_LLD).
cc, cxx, lld = None, None, None
cflags = []
cxxflags = []
ldflags = []
targets = 'AArch64;ARM;Mips;PowerPC;SystemZ;WebAssembly;X86'
projects = 'clang;compiler-rt;lld;chrometools;clang-tools-extra'
if sys.platform == 'darwin':
# clang needs libc++, else -stdlib=libc++ won't find includes
# (this is needed for bootstrap builds and for building the fuchsia runtime)
projects += ';libcxx'
base_cmake_args = [
'-GNinja',
'-DCMAKE_BUILD_TYPE=Release',
'-DLLVM_ENABLE_ASSERTIONS=%s' % ('OFF' if args.disable_asserts else 'ON'),
'-DLLVM_ENABLE_PROJECTS=' + projects,
'-DLLVM_TARGETS_TO_BUILD=' + targets,
'-DLLVM_ENABLE_PIC=OFF',
'-DLLVM_ENABLE_UNWIND_TABLES=OFF',
'-DLLVM_ENABLE_TERMINFO=OFF',
'-DLLVM_ENABLE_Z3_SOLVER=OFF',
'-DCLANG_PLUGIN_SUPPORT=OFF',
'-DCLANG_ENABLE_STATIC_ANALYZER=OFF',
'-DCLANG_ENABLE_ARCMT=OFF',
'-DBUG_REPORT_URL=' + BUG_REPORT_URL,
# Don't run Go bindings tests; PGO makes them confused.
'-DLLVM_INCLUDE_GO_TESTS=OFF',
# TODO(crbug.com/1113475): Update binutils.
'-DENABLE_X86_RELAX_RELOCATIONS=NO',
# See crbug.com/1126219: Use native symbolizer instead of DIA
'-DLLVM_ENABLE_DIA_SDK=OFF',
]
if args.gcc_toolchain:
# Use the specified gcc installation for building.
cc = os.path.join(args.gcc_toolchain, 'bin', 'gcc')
cxx = os.path.join(args.gcc_toolchain, 'bin', 'g++')
if not os.access(cc, os.X_OK):
print('Invalid --gcc-toolchain: ' + args.gcc_toolchain)
return 1
base_cmake_args += [
'-DLLVM_LOCAL_RPATH=' + os.path.join(args.gcc_toolchain, 'lib64')
]
if sys.platform == 'darwin':
# For libc++, we only want the headers.
base_cmake_args.extend([
'-DLIBCXX_ENABLE_SHARED=OFF',
'-DLIBCXX_ENABLE_STATIC=OFF',
'-DLIBCXX_INCLUDE_TESTS=OFF',
'-DLIBCXX_ENABLE_EXPERIMENTAL_LIBRARY=OFF',
])
if args.gcc_toolchain:
# Force compiler-rt tests to use our gcc toolchain (including libstdc++.so)
# because the one on the host may be too old.
base_cmake_args.append(
'-DCOMPILER_RT_TEST_COMPILER_CFLAGS=--gcc-toolchain=' +
args.gcc_toolchain + ' -Wl,-rpath,' +
os.path.join(args.gcc_toolchain, 'lib64') + ' -Wl,-rpath,' +
os.path.join(args.gcc_toolchain, 'lib32'))
if sys.platform == 'win32':
base_cmake_args.append('-DLLVM_USE_CRT_RELEASE=MT')
# Require zlib compression.
zlib_dir = AddZlibToPath()
cflags.append('-I' + zlib_dir)
cxxflags.append('-I' + zlib_dir)
ldflags.append('-LIBPATH:' + zlib_dir)
# Use rpmalloc. For faster ThinLTO linking.
rpmalloc_dir = DownloadRPMalloc()
base_cmake_args.append('-DLLVM_INTEGRATED_CRT_ALLOC=' + rpmalloc_dir)
if sys.platform != 'win32':
# libxml2 is required by the Win manifest merging tool used in cross-builds.
base_cmake_args.append('-DLLVM_ENABLE_LIBXML2=FORCE_ON')
if args.bootstrap:
print('Building bootstrap compiler')
if os.path.exists(LLVM_BOOTSTRAP_DIR):
RmTree(LLVM_BOOTSTRAP_DIR)
EnsureDirExists(LLVM_BOOTSTRAP_DIR)
os.chdir(LLVM_BOOTSTRAP_DIR)
projects = 'clang'
if args.pgo:
# Need libclang_rt.profile
projects += ';compiler-rt'
if sys.platform != 'darwin':
projects += ';lld'
if sys.platform == 'darwin':
# Need libc++ and compiler-rt for the bootstrap compiler on mac.
projects += ';libcxx;compiler-rt'
bootstrap_targets = 'X86'
if sys.platform == 'darwin':
# Need ARM and AArch64 for building the ios clang_rt.
bootstrap_targets += ';ARM;AArch64'
bootstrap_args = base_cmake_args + [
'-DLLVM_TARGETS_TO_BUILD=' + bootstrap_targets,
'-DLLVM_ENABLE_PROJECTS=' + projects,
'-DCMAKE_INSTALL_PREFIX=' + LLVM_BOOTSTRAP_INSTALL_DIR,
'-DCMAKE_C_FLAGS=' + ' '.join(cflags),
'-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
'-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
'-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
'-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
# Ignore args.disable_asserts for the bootstrap compiler.
'-DLLVM_ENABLE_ASSERTIONS=ON',
]
if sys.platform == 'darwin':
# On macOS, the bootstrap toolchain needs to have compiler-rt because
# dsymutil's link needs libclang_rt.osx.a. Only the x86_64 osx
# libraries are needed though, and only libclang_rt (i.e.
# COMPILER_RT_BUILD_BUILTINS).
bootstrap_args.extend([
'-DDARWIN_osx_ARCHS=x86_64',
'-DCOMPILER_RT_BUILD_BUILTINS=ON',
'-DCOMPILER_RT_BUILD_CRT=OFF',
'-DCOMPILER_RT_BUILD_LIBFUZZER=OFF',
'-DCOMPILER_RT_BUILD_MEMPROF=OFF',
'-DCOMPILER_RT_BUILD_SANITIZERS=OFF',
'-DCOMPILER_RT_BUILD_XRAY=OFF',
'-DCOMPILER_RT_ENABLE_IOS=OFF',
'-DCOMPILER_RT_ENABLE_WATCHOS=OFF',
'-DCOMPILER_RT_ENABLE_TVOS=OFF',
])
elif args.pgo:
# PGO needs libclang_rt.profile but none of the other compiler-rt stuff.
bootstrap_args.extend([
'-DCOMPILER_RT_BUILD_BUILTINS=OFF',
'-DCOMPILER_RT_BUILD_CRT=OFF',
'-DCOMPILER_RT_BUILD_LIBFUZZER=OFF',
'-DCOMPILER_RT_BUILD_MEMPROF=OFF',
'-DCOMPILER_RT_BUILD_PROFILE=ON',
'-DCOMPILER_RT_BUILD_SANITIZERS=OFF',
'-DCOMPILER_RT_BUILD_XRAY=OFF',
])
if cc is not None: bootstrap_args.append('-DCMAKE_C_COMPILER=' + cc)
if cxx is not None: bootstrap_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
if lld is not None: bootstrap_args.append('-DCMAKE_LINKER=' + lld)
RunCommand(['cmake'] + bootstrap_args + [os.path.join(LLVM_DIR, 'llvm')],
msvc_arch='x64')
CopyLibstdcpp(args, LLVM_BOOTSTRAP_DIR)
CopyLibstdcpp(args, LLVM_BOOTSTRAP_INSTALL_DIR)
RunCommand(['ninja'], msvc_arch='x64')
if args.run_tests:
RunCommand(['ninja', 'check-all'], msvc_arch='x64')
RunCommand(['ninja', 'install'], msvc_arch='x64')
if sys.platform == 'win32':
cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
lld = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'lld-link.exe')
# CMake has a hard time with backslashes in compiler paths:
# https://stackoverflow.com/questions/13050827
cc = cc.replace('\\', '/')
cxx = cxx.replace('\\', '/')
lld = lld.replace('\\', '/')
else:
cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang')
cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang++')
if sys.platform.startswith('linux'):
base_cmake_args.append('-DLLVM_ENABLE_LLD=ON')
if args.gcc_toolchain:
# Tell the bootstrap compiler where to find the standard library headers
# and shared object files.
cflags.append('--gcc-toolchain=' + args.gcc_toolchain)
cxxflags.append('--gcc-toolchain=' + args.gcc_toolchain)
print('Bootstrap compiler installed.')
if args.pgo:
print('Building instrumented compiler')
if os.path.exists(LLVM_INSTRUMENTED_DIR):
RmTree(LLVM_INSTRUMENTED_DIR)
EnsureDirExists(LLVM_INSTRUMENTED_DIR)
os.chdir(LLVM_INSTRUMENTED_DIR)
projects = 'clang'
if sys.platform == 'darwin':
projects += ';libcxx;compiler-rt'
instrument_args = base_cmake_args + [
'-DLLVM_ENABLE_PROJECTS=' + projects,
'-DCMAKE_C_FLAGS=' + ' '.join(cflags),
'-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
'-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
'-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
'-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
# Build with instrumentation.
'-DLLVM_BUILD_INSTRUMENTED=IR',
]
# Build with the bootstrap compiler.
if cc is not None: instrument_args.append('-DCMAKE_C_COMPILER=' + cc)
if cxx is not None: instrument_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
if lld is not None: instrument_args.append('-DCMAKE_LINKER=' + lld)
RunCommand(['cmake'] + instrument_args + [os.path.join(LLVM_DIR, 'llvm')],
msvc_arch='x64')
CopyLibstdcpp(args, LLVM_INSTRUMENTED_DIR)
RunCommand(['ninja'], msvc_arch='x64')
print('Instrumented compiler built.')
# Train by building some C++ code.
#
# pgo_training-1.ii is a preprocessed (on Linux) version of
# src/third_party/blink/renderer/core/layout/layout_object.cc, selected
# because it's a large translation unit in Blink, which is normally the
# slowest part of Chromium to compile. Using this, we get ~20% shorter
# build times for Linux, Android, and Mac, which is also what we got when
# training by actually building a target in Chromium. (For comparison, a
# C++-y "Hello World" program only resulted in 14% faster builds.)
# See https://crbug.com/966403#c16 for all numbers.
#
# Although the training currently only exercises Clang, it does involve LLVM
# internals, and so LLD also benefits when used for ThinLTO links.
#
# NOTE: Tidy uses binaries built with this profile, but doesn't seem to
# gain much from it. If tidy's execution time becomes a concern, it might
# be good to investigate that.
#
# TODO(hans): Enhance the training, perhaps by including preprocessed code
# from more platforms, and by doing some linking so that lld can benefit
# from PGO as well. Perhaps the training could be done asynchronously by
# dedicated buildbots that upload profiles to the cloud.
training_source = 'pgo_training-1.ii'
with open(training_source, 'wb') as f:
DownloadUrl(CDS_URL + '/' + training_source, f)
train_cmd = [os.path.join(LLVM_INSTRUMENTED_DIR, 'bin', 'clang++'),
'-target', 'x86_64-unknown-unknown', '-O2', '-g', '-std=c++14',
'-fno-exceptions', '-fno-rtti', '-w', '-c', training_source]
if sys.platform == 'darwin':
train_cmd.extend(['-stdlib=libc++', '-isysroot', isysroot])
RunCommand(train_cmd, msvc_arch='x64')
# Merge profiles.
profdata = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'llvm-profdata')
RunCommand([profdata, 'merge', '-output=' + LLVM_PROFDATA_FILE] +
glob.glob(os.path.join(LLVM_INSTRUMENTED_DIR, 'profiles',
'*.profraw')), msvc_arch='x64')
print('Profile generated.')
compiler_rt_args = [
'-DCOMPILER_RT_BUILD_CRT=OFF',
'-DCOMPILER_RT_BUILD_LIBFUZZER=OFF',
'-DCOMPILER_RT_BUILD_MEMPROF=OFF',
'-DCOMPILER_RT_BUILD_PROFILE=ON',
'-DCOMPILER_RT_BUILD_SANITIZERS=ON',
'-DCOMPILER_RT_BUILD_XRAY=OFF',
]
if sys.platform == 'darwin':
compiler_rt_args.extend([
'-DCOMPILER_RT_BUILD_BUILTINS=ON',
'-DCOMPILER_RT_ENABLE_IOS=ON',
'-DCOMPILER_RT_ENABLE_WATCHOS=OFF',
'-DCOMPILER_RT_ENABLE_TVOS=OFF',
# armv7 is A5 and earlier, armv7s is A6+ (2012 and later, before 64-bit
# iPhones). armv7k is Apple Watch, which we don't need.
'-DDARWIN_ios_ARCHS=armv7;armv7s;arm64',
'-DDARWIN_iossim_ARCHS=i386;x86_64;arm64',
])
if args.bootstrap:
# mac/arm64 needs MacOSX11.0.sdk. System Xcode (+ SDK) may be something
# else, so use the hermetic Xcode.
# Options:
# - temporarily set system Xcode to Xcode 12 beta while running this
# script, (cf build/swarming_xcode_install.py, but it looks unused)
# - use Xcode 12 beta for everything on tot bots, only need to fuzz with
# scripts/slave/recipes/chromium_upload_clang.py then (but now the
# chrome/ios build will use the 11.0 SDK too and we'd be on the hook for
# keeping it green -- if it's currently green, who knows)
# - pass flags to cmake to try to coax it into using Xcode 12 beta for the
# LLVM build without it being system Xcode.
#
# The last option seems best, so let's go with that. We need to pass
# -isysroot to the SDK and -B to the /usr/bin so that the new ld64 is
# used.
# The compiler-rt build overrides -isysroot flags set via cflags, and we
# only need to use the 11 SDK for the compiler-rt build. So set only
# DARWIN_macosx_CACHED_SYSROOT to the 11 SDK and use the regular SDK
# for the rest of the build. (The new ld is used for all links.)
sys.path.insert(1, os.path.join(CHROMIUM_DIR, 'build'))
import mac_toolchain
LLVM_XCODE = os.path.join(THIRD_PARTY_DIR, 'llvm-xcode')
mac_toolchain.InstallXcodeBinaries(LLVM_XCODE)
isysroot_11 = os.path.join(LLVM_XCODE, 'Contents', 'Developer',
'Platforms', 'MacOSX.platform', 'Developer',
'SDKs', 'MacOSX11.1.sdk')
xcode_bin = os.path.join(LLVM_XCODE, 'Contents', 'Developer',
'Toolchains', 'XcodeDefault.xctoolchain', 'usr',
'bin')
# Include an arm64 slice for libclang_rt.osx.a. This requires using
# MacOSX11.x.sdk (via -isysroot, via DARWIN_macosx_CACHED_SYSROOT) and
# the new ld, via -B
compiler_rt_args.extend([
# We don't need 32-bit intel support for macOS, we only ship 64-bit.
'-DDARWIN_osx_ARCHS=arm64;x86_64',
'-DDARWIN_macosx_CACHED_SYSROOT=' + isysroot_11,
])
ldflags += ['-B', xcode_bin]
else:
compiler_rt_args.extend(['-DDARWIN_osx_ARCHS=x86_64'])
else:
compiler_rt_args.append('-DCOMPILER_RT_BUILD_BUILTINS=OFF')
# LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is
# needed, on OS X it requires libc++. clang only automatically links to libc++
# when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run
# on OS X versions as old as 10.7.
deployment_target = ''
if sys.platform == 'darwin' and args.bootstrap:
# When building on 10.9, /usr/include usually doesn't exist, and while
# Xcode's clang automatically sets a sysroot, self-built clangs don't.
cflags = ['-isysroot', isysroot]
cxxflags = ['-stdlib=libc++'] + cflags
ldflags += ['-stdlib=libc++']
deployment_target = '10.7'
# If building at head, define a macro that plugins can use for #ifdefing
# out code that builds at head, but not at CLANG_REVISION or vice versa.
if args.llvm_force_head_revision:
cflags += ['-DLLVM_FORCE_HEAD_REVISION']
cxxflags += ['-DLLVM_FORCE_HEAD_REVISION']
# Build PDBs for archival on Windows. Don't use RelWithDebInfo since it
# has different optimization defaults than Release.
# Also disable stack cookies (/GS-) for performance.
if sys.platform == 'win32':
cflags += ['/Zi', '/GS-']
cxxflags += ['/Zi', '/GS-']
ldflags += ['/DEBUG', '/OPT:REF', '/OPT:ICF']
deployment_env = None
if deployment_target:
deployment_env = os.environ.copy()
deployment_env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target
print('Building final compiler.')
default_tools = ['plugins', 'blink_gc_plugin', 'translation_unit']
chrome_tools = list(set(default_tools + args.extra_tools))
if cc is not None: base_cmake_args.append('-DCMAKE_C_COMPILER=' + cc)
if cxx is not None: base_cmake_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
if lld is not None: base_cmake_args.append('-DCMAKE_LINKER=' + lld)
cmake_args = base_cmake_args + compiler_rt_args + [
'-DCMAKE_C_FLAGS=' + ' '.join(cflags),
'-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
'-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
'-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
'-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
'-DCMAKE_INSTALL_PREFIX=' + LLVM_BUILD_DIR,
'-DCHROMIUM_TOOLS_SRC=%s' % os.path.join(CHROMIUM_DIR, 'tools', 'clang'),
'-DCHROMIUM_TOOLS=%s' % ';'.join(chrome_tools)]
if args.pgo:
cmake_args.append('-DLLVM_PROFDATA_FILE=' + LLVM_PROFDATA_FILE)
if args.thinlto:
cmake_args.append('-DLLVM_ENABLE_LTO=Thin')
if sys.platform == 'win32':
cmake_args.append('-DLLVM_ENABLE_ZLIB=FORCE_ON')
if sys.platform == 'darwin':
cmake_args += ['-DCOMPILER_RT_ENABLE_IOS=ON',
'-DSANITIZER_MIN_OSX_VERSION=10.7']
# TODO(crbug.com/962988): Use -DLLVM_EXTERNAL_PROJECTS instead.
CreateChromeToolsShim()
if os.path.exists(LLVM_BUILD_DIR):
RmTree(LLVM_BUILD_DIR)
EnsureDirExists(LLVM_BUILD_DIR)
os.chdir(LLVM_BUILD_DIR)
RunCommand(['cmake'] + cmake_args + [os.path.join(LLVM_DIR, 'llvm')],
msvc_arch='x64', env=deployment_env)
CopyLibstdcpp(args, LLVM_BUILD_DIR)
RunCommand(['ninja'], msvc_arch='x64')
if chrome_tools:
# If any Chromium tools were built, install those now.
RunCommand(['ninja', 'cr-install'], msvc_arch='x64')
VerifyVersionOfBuiltClangMatchesVERSION()
VerifyZlibSupport()
if sys.platform == 'win32':
platform = 'windows'
elif sys.platform == 'darwin':
platform = 'darwin'
else:
assert sys.platform.startswith('linux')
platform = 'linux'
rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
RELEASE_VERSION, 'lib', platform)
# Do an out-of-tree build of compiler-rt for 32-bit Win clang_rt.profile.lib.
if sys.platform == 'win32':
compiler_rt_build_dir = os.path.join(LLVM_BUILD_DIR, 'compiler-rt')
if os.path.isdir(compiler_rt_build_dir):
RmTree(compiler_rt_build_dir)
os.makedirs(compiler_rt_build_dir)
os.chdir(compiler_rt_build_dir)
if args.bootstrap:
# The bootstrap compiler produces 64-bit binaries by default.
cflags += ['-m32']
cxxflags += ['-m32']
compiler_rt_args = base_cmake_args + [
'-DCMAKE_C_FLAGS=' + ' '.join(cflags),
'-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
'-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
'-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
'-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
'-DCOMPILER_RT_BUILD_BUILTINS=OFF',
'-DCOMPILER_RT_BUILD_CRT=OFF',
'-DCOMPILER_RT_BUILD_LIBFUZZER=OFF',
'-DCOMPILER_RT_BUILD_MEMPROF=OFF',
'-DCOMPILER_RT_BUILD_PROFILE=ON',
'-DCOMPILER_RT_BUILD_SANITIZERS=OFF',
'-DCOMPILER_RT_BUILD_XRAY=OFF',
]
RunCommand(['cmake'] + compiler_rt_args +
[os.path.join(LLVM_DIR, 'llvm')],
msvc_arch='x86', env=deployment_env)
RunCommand(['ninja', 'compiler-rt'], msvc_arch='x86')
# Copy select output to the main tree.
rt_lib_src_dir = os.path.join(compiler_rt_build_dir, 'lib', 'clang',
RELEASE_VERSION, 'lib', platform)
# Static and dynamic libraries:
CopyDirectoryContents(rt_lib_src_dir, rt_lib_dst_dir)
if args.with_android:
# TODO(thakis): Now that the NDK uses clang, try to build all archs in
# one LLVM build instead of building 3 times.
toolchain_dir = ANDROID_NDK_DIR + '/toolchains/llvm/prebuilt/linux-x86_64'
for target_arch in ['aarch64', 'arm', 'i686']:
# Build compiler-rt runtimes needed for Android in a separate build tree.
build_dir = os.path.join(LLVM_BUILD_DIR, 'android-' + target_arch)
if not os.path.exists(build_dir):
os.mkdir(os.path.join(build_dir))
os.chdir(build_dir)
target_triple = target_arch
if target_arch == 'arm':
target_triple = 'armv7'
api_level = '21' if target_arch == 'aarch64' else '19'
target_triple += '-linux-android' + api_level
cflags = [
'--target=' + target_triple,
'--sysroot=%s/sysroot' % toolchain_dir,
'--gcc-toolchain=' + toolchain_dir,
# android_ndk/toolchains/llvm/prebuilt/linux-x86_64/aarch64-linux-android/bin/ld
# depends on a newer version of libxml2.so than what's available on
# the bots. To make things work, use our just-built lld as linker.
'-fuse-ld=lld',
# Clang defaults to compiler-rt when targeting android after
# a478b0a199f4. Stick with libgcc for now. (crbug.com/1184398).
'--rtlib=libgcc',
]
android_args = base_cmake_args + [
'-DCMAKE_C_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang'),
'-DCMAKE_CXX_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang++'),
'-DLLVM_CONFIG_PATH=' + os.path.join(LLVM_BUILD_DIR, 'bin/llvm-config'),
'-DCMAKE_C_FLAGS=' + ' '.join(cflags),
'-DCMAKE_CXX_FLAGS=' + ' '.join(cflags),
'-DCMAKE_ASM_FLAGS=' + ' '.join(cflags),
'-DCOMPILER_RT_BUILD_BUILTINS=OFF',
'-DCOMPILER_RT_BUILD_CRT=OFF',
'-DCOMPILER_RT_BUILD_LIBFUZZER=OFF',
'-DCOMPILER_RT_BUILD_MEMPROF=OFF',
'-DCOMPILER_RT_BUILD_PROFILE=ON',
'-DCOMPILER_RT_BUILD_SANITIZERS=ON',
'-DCOMPILER_RT_BUILD_XRAY=OFF',
'-DSANITIZER_CXX_ABI=libcxxabi',
'-DCMAKE_SHARED_LINKER_FLAGS=-Wl,-u__cxa_demangle',
'-DANDROID=1']
RunCommand(['cmake'] + android_args + [COMPILER_RT_DIR])
# We use ASan i686 build for fuzzing.
libs_want = ['lib/linux/libclang_rt.asan-{0}-android.so']
if target_arch in ['aarch64', 'arm']:
libs_want += [
'lib/linux/libclang_rt.ubsan_standalone-{0}-android.so',
'lib/linux/libclang_rt.profile-{0}-android.a',
]
if target_arch == 'aarch64':
libs_want += ['lib/linux/libclang_rt.hwasan-{0}-android.so']
libs_want = [lib.format(target_arch) for lib in libs_want]
RunCommand(['ninja'] + libs_want)
# And copy them into the main build tree.
for p in libs_want:
shutil.copy(p, rt_lib_dst_dir)
if args.with_fuchsia:
# Fuchsia links against libclang_rt.builtins-<arch>.a instead of libgcc.a.
for target_arch in ['aarch64', 'x86_64']:
fuchsia_arch_name = {'aarch64': 'arm64', 'x86_64': 'x64'}[target_arch]
toolchain_dir = os.path.join(
FUCHSIA_SDK_DIR, 'arch', fuchsia_arch_name, 'sysroot')
# Build clang_rt runtime for Fuchsia in a separate build tree.
build_dir = os.path.join(LLVM_BUILD_DIR, 'fuchsia-' + target_arch)
if not os.path.exists(build_dir):
os.mkdir(os.path.join(build_dir))
os.chdir(build_dir)
target_spec = target_arch + '-fuchsia'
# TODO(thakis): Might have to pass -B here once sysroot contains
# binaries (e.g. gas for arm64?)
fuchsia_args = base_cmake_args + [
'-DCMAKE_C_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang'),
'-DCMAKE_CXX_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang++'),
'-DCMAKE_LINKER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang'),
'-DCMAKE_AR=' + os.path.join(LLVM_BUILD_DIR, 'bin/llvm-ar'),
'-DLLVM_CONFIG_PATH=' + os.path.join(LLVM_BUILD_DIR, 'bin/llvm-config'),
'-DCMAKE_SYSTEM_NAME=Fuchsia',
'-DCMAKE_C_COMPILER_TARGET=%s-fuchsia' % target_arch,
'-DCMAKE_ASM_COMPILER_TARGET=%s-fuchsia' % target_arch,
'-DCOMPILER_RT_BUILD_BUILTINS=ON',
'-DCOMPILER_RT_BUILD_CRT=OFF',
'-DCOMPILER_RT_BUILD_LIBFUZZER=OFF',
'-DCOMPILER_RT_BUILD_MEMPROF=OFF',
'-DCOMPILER_RT_BUILD_PROFILE=OFF',
'-DCOMPILER_RT_BUILD_SANITIZERS=OFF',
'-DCOMPILER_RT_BUILD_XRAY=OFF',
'-DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON',
'-DCMAKE_SYSROOT=%s' % toolchain_dir,
# TODO(thakis|scottmg): Use PER_TARGET_RUNTIME_DIR for all platforms.
# https://crbug.com/882485.
'-DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=ON',
# These are necessary because otherwise CMake tries to build an
# executable to test to see if the compiler is working, but in doing so,
# it links against the builtins.a that we're about to build.
'-DCMAKE_C_COMPILER_WORKS=ON',
'-DCMAKE_ASM_COMPILER_WORKS=ON',
]
RunCommand(['cmake'] +
fuchsia_args +
[os.path.join(COMPILER_RT_DIR, 'lib', 'builtins')])
builtins_a = 'libclang_rt.builtins.a'
RunCommand(['ninja', builtins_a])
# And copy it into the main build tree.
fuchsia_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
RELEASE_VERSION, 'lib', target_spec)
if not os.path.exists(fuchsia_lib_dst_dir):
os.makedirs(fuchsia_lib_dst_dir)
CopyFile(os.path.join(build_dir, 'lib', target_spec, builtins_a),
fuchsia_lib_dst_dir)
# Build the Fuchsia profile runtime.
if target_arch == 'x86_64':
fuchsia_args.extend([
'-DCOMPILER_RT_BUILD_BUILTINS=OFF',
'-DCOMPILER_RT_BUILD_PROFILE=ON',
'-DCMAKE_CXX_COMPILER_TARGET=%s-fuchsia' % target_arch,
'-DCMAKE_CXX_COMPILER_WORKS=ON',
])
profile_build_dir = os.path.join(LLVM_BUILD_DIR,
'fuchsia-profile-' + target_arch)
if not os.path.exists(profile_build_dir):
os.mkdir(os.path.join(profile_build_dir))
os.chdir(profile_build_dir)
RunCommand(['cmake'] +
fuchsia_args +
[COMPILER_RT_DIR])
profile_a = 'libclang_rt.profile.a'
RunCommand(['ninja', profile_a])
CopyFile(os.path.join(profile_build_dir, 'lib', target_spec, profile_a),
fuchsia_lib_dst_dir)
# Run tests.
if args.run_tests or args.llvm_force_head_revision:
RunCommand(['ninja', '-C', LLVM_BUILD_DIR, 'cr-check-all'], msvc_arch='x64')
if args.run_tests:
test_targets = [ 'check-all' ]
if sys.platform == 'darwin':
# TODO(thakis): Run check-all on Darwin too, https://crbug.com/959361
test_targets = [ 'check-llvm', 'check-clang', 'check-lld' ]
RunCommand(['ninja', '-C', LLVM_BUILD_DIR] + test_targets, msvc_arch='x64')
if sys.platform == 'darwin':
for dylib in glob.glob(os.path.join(rt_lib_dst_dir, '*.dylib')):
# Fix LC_ID_DYLIB for the ASan dynamic libraries to be relative to
# @executable_path.
# Has to happen after running tests.
# TODO(glider): this is transitional. We'll need to fix the dylib
# name either in our build system, or in Clang. See also
# http://crbug.com/344836.
subprocess.call(['install_name_tool', '-id',
'@executable_path/' + os.path.basename(dylib), dylib])
WriteStampFile(PACKAGE_VERSION, STAMP_FILE)
WriteStampFile(PACKAGE_VERSION, FORCE_HEAD_REVISION_FILE)
print('Clang build was successful.')
return 0