def ConfigureGCCProject(arch, project, cfg, workpath, inspath):
# configure does not always have +x
filepath = os.path.abspath(os.path.join(workpath, cfg[0]))
st_info = os.stat(filepath)
os.chmod(filepath, st_info.st_mode | stat.S_IEXEC)
env = os.environ
newpath = GetToolchainPath(arch, 'bionic', 'bin') + ':' + env['PATH']
proj = '%s %s' % (project, arch)
setpath = ['/usr/bin/env', 'PATH=' + newpath]
# Check if config changed or script is new
config_path = os.path.join(workpath, 'config.info')
updated = UpdateText(config_path, ' '.join(cfg))
updated |= NeedsUpdate(config_path, BUILD_SCRIPT)
if updated:
print 'Configure ' + proj
if process.Run(setpath + cfg,
cwd=workpath,
env=env,
outfile=sys.stdout):
raise RuntimeError('Failed to configure %s.\n' % proj)
else:
print 'Reusing config for %s.' % proj
def FetchAndBuildGCC():
if 'arm' in ARCHES:
tc_args = ['-y', '--no-use-remote-cache', 'gcc_libs_arm']
toolchain_main.PackageBuilder(toolchain_build.PACKAGES, tc_args).Main()
if 'x86' in ARCHES:
process.Run(['make', 'sync'], cwd=os.path.join(NATIVE_CLIENT, 'tools'))
def ArchiveAndUpload(version, zipname, zippath, packages_file):
sys.stdout.flush()
print >>sys.stderr, '@@@BUILD_STEP archive_and_upload@@@'
bucket_path = 'nativeclient-archive2/toolchain/%s' % version
gsd_store = pynacl.gsd_storage.GSDStorage(bucket_path, [bucket_path])
zipname = os.path.join(TOOLCHAIN_BUILD_OUT, zipname)
try:
os.remove(zipname)
except:
pass
# Archive the zippath to the zipname.
if process.Run(['tar', '-czf', zipname, zippath],
cwd=TOOLCHAIN_BUILD_OUT,
outfile=sys.stdout):
raise RuntimeError('Failed to zip %s from %s.\n' % (zipname, zippath))
# Create Zip Hash file using the hash of the zip file.
hashzipname = zipname + '.sha1hash'
hashval = pynacl.hashing_tools.HashFileContents(zipname)
with open(hashzipname, 'w') as f:
f.write(hashval)
# Upload the Zip file.
zipurl = gsd_store.PutFile(zipname, os.path.basename(zipname))
sys.stdout.flush()
print >>sys.stderr, ('@@@STEP_LINK@download (%s)@%s@@@' %
(os.path.basename(zipname), zipurl))
# Upload the Zip Hash file.
hashurl = gsd_store.PutFile(hashzipname, os.path.basename(hashzipname))
sys.stdout.flush()
print >>sys.stderr, ('@@@STEP_LINK@download (%s)@%s@@@' %
(os.path.basename(hashzipname), hashurl))
# Create a package info file for the nacl_arm_bionic package.
archive_desc = archive_info.ArchiveInfo(name=os.path.basename(zipname),
archive_hash=hashval,
tar_src_dir='linux_arm_bionic',
url=zipurl)
package_desc = package_info.PackageInfo()
package_desc.AppendArchive(archive_desc)
os_name = pynacl.platform.GetOS()
arch_name = pynacl.platform.GetArch()
package_info_file = os.path.join(TOOLCHAIN_BUILD_OUT,
'packages',
'%s_%s' % (os_name, arch_name),
'nacl_arm_bionic.json')
package_desc.SavePackageFile(package_info_file)
# If packages_file is specified, write out our packages file of 1 package.
if packages_file:
with open(packages_file, 'wt') as f:
f.write(package_info_file)
def MakeBionicProject(project, targets=[], clobber=False):
arch = 'arm'
paths = GetProjectPaths(arch, project)
workpath = paths['work']
inspath = paths['ins']
targetlist = ' '.join(targets)
print 'Building %s for %s at %s %s.' % (project, arch, workpath, targetlist)
if clobber:
args = ['make', '-j12', 'V=1', 'clean']
if process.Run(args, cwd=workpath, outfile=sys.stdout):
raise RuntimeError('Failed to clean %s for %s.\n' % (project, arch))
args = ['make', '-j12', 'V=1'] + targets
if process.Run(args, cwd=workpath, outfile=sys.stdout):
raise RuntimeError('Failed to build %s for %s.\n' % (project, arch))
print 'Done with %s for %s.\n' % (project, arch)
def ArchiveAndUpload(version, zipname, zippath):
if 'BUILDBOT_BUILDERNAME' in os.environ:
GSUTIL = '../buildbot/gsutil.sh'
else:
GSUTIL = 'gsutil'
GSUTIL_ARGS = [GSUTIL, 'cp', '-a', 'public-read']
GSUTIL_PATH = 'gs://nativeclient-archive2/toolchain'
urldir = os.path.join(GSUTIL_PATH, version)
zipurl = os.path.join(urldir, zipname)
zipname = os.path.join(TOOLCHAIN_BUILD_OUT, zipname)
try:
os.remove(zipname)
except:
pass
sys.stdout.flush()
print >> sys.stderr, '@@@STEP_LINK@download@%s@@@' % urldir
if process.Run(['tar', '-czf', zipname, zippath],
cwd=TOOLCHAIN,
outfile=sys.stdout):
raise RuntimeError('Failed to zip %s from %s.\n' % (zipname, zippath))
hashzipname = zipname + '.sha1hash'
hashzipurl = zipurl + '.sha1hash'
hashval = pynacl.hashing_tools.HashFileContents(zipname)
with open(hashzipname, 'w') as f:
f.write(hashval)
if process.Run(GSUTIL_ARGS + [zipname, zipurl],
cwd=TOOLCHAIN_BUILD,
outfile=sys.stdout):
err = 'Failed to upload zip %s to %s.\n' % (zipname, zipurl)
raise RuntimeError(err)
if process.Run(GSUTIL_ARGS + [hashzipname, hashzipurl],
cwd=TOOLCHAIN_BUILD,
outfile=sys.stdout):
err = 'Failed to upload hash %s to %s.\n' % (hashzipname, hashzipurl)
raise RuntimeError(err)
def ConfigureAndInstallForGCC(arch, project, cfg, workpath, inspath):
# configure does not always have +x
filepath = os.path.abspath(os.path.join(workpath, cfg[0]))
st_info = os.stat(filepath)
os.chmod(filepath, st_info.st_mode | stat.S_IEXEC)
env = os.environ
if arch == 'arm':
newpath = os.path.join(ARM_BIONIC, 'bin') + ':' + env['PATH']
else:
newpath = os.path.join(X86_BIONIC, 'bin') + ':' + env['PATH']
proj = '%s %s' % (project, arch)
setpath = ['/usr/bin/env', 'PATH=' + newpath]
# Check if config changed or script is new
config_path = os.path.join(workpath, 'config.info')
updated = UpdateText(config_path, ' '.join(cfg))
updated |= NeedsUpdate(config_path, BUILD_SCRIPT)
if updated:
print 'Configure ' + proj
if process.Run(setpath + cfg,
cwd=workpath,
env=env,
outfile=sys.stdout):
raise RuntimeError('Failed to configure %s.\n' % proj)
else:
print 'Reusing config for %s.' % proj
print 'Make ' + proj
if process.Run(setpath + ['make', '-j16', 'V=1'],
cwd=workpath,
outfile=sys.stdout):
raise RuntimeError('Failed to build %s.\n' % proj)
print 'Install ' + proj
if process.Run(setpath + ['make', '-j16', 'install', 'V=1'],
cwd=workpath,
outfile=sys.stdout):
raise RuntimeError('Failed to install %s.\n' % proj)
print 'Done ' + proj
def ConfigureAndBuild(arch, project, workpath, inspath, tcpath=None):
# Create project for CRTx and LIBC files
print 'Configure %s for %s.\n' % (project, arch)
srcpath = os.path.join(TOOLCHAIN_BUILD_SRC, project)
dstpath = ReplaceArch(os.path.join(workpath, '$NACL-nacl', project), arch)
libpath = ReplaceArch(os.path.join(inspath, '$NACL-nacl', 'lib'), arch)
CreateProject(arch, srcpath, dstpath, libpath)
print 'Building %s for %s at %s.' % (project, arch, dstpath)
if process.Run(['make', '-j12', 'V=1'], cwd=dstpath, outfile=sys.stdout):
raise RuntimeError('Failed to build %s for %s.\n' % (project, arch))
if tcpath:
UpdateFromTo(inspath, tcpath)
print 'Done with %s for %s.\n' % (project, arch)
def MakeGCCProject(arch, project, workpath, targets=[]):
env = os.environ
newpath = GetBionicBuildPath(arch, 'bin') + ':' + env['PATH']
proj = '%s %s' % (project, arch)
setpath = ['/usr/bin/env', 'PATH=' + newpath]
if targets:
proj = project = ': ' + ' '.join(targets)
else:
proj = project
print 'Make ' + proj
if process.Run(setpath + ['make', '-j16', 'V=1'] + targets,
cwd=workpath, outfile=sys.stdout):
raise RuntimeError('Failed to build %s.\n' % proj)
print 'Done ' + proj
def CreateBasicToolchain():
# Create a toolchain directory containing only the toolchain binaries and
# basic files line nacl_arm_macros.s.
arch = 'arm'
UpdateFromTo(
GetToolchainPath(arch, 'newlib'),
GetToolchainPath(arch, 'bionic'),
filters=['*arm-nacl/include*', '*arm-nacl/lib*', '*.a', '*.o'])
UpdateFromTo(GetToolchainPath(arch, 'newlib'),
GetToolchainPath(arch, 'bionic'),
paterns=['*.s'])
# Static build uses:
# crt1.o crti.o 4.8.2/crtbeginT.o ... 4.8.2/crtend.o crtn.o
# -shared build uses:
# crti.o 4.8.2/crtbeginS.o ... 4.8.2/crtendS.o crtn.o crtn.o
# However we only provide a crtbegin(S) and crtend(S)
EMPTY = """/*
* This is a dummy linker script.
* libnacl.a, libcrt_common.a, crt0.o crt1.o crti.o and crtn.o are all
* empty. Instead all the work is done by crtbegin(S).o and crtend(S).o and
* the bionic libc. These are provided for compatability with the newlib
* toolchain binaries.
*/"""
EMPTY_FILES = [
'crt0.o', 'crt1.o', 'crti.o', 'crtn.o', 'libnacl.a', 'libcrt_common.a',
'libpthread.a'
]
# Bionic uses the following include paths
BIONIC_PAIRS = [
('bionic/libc/arch-nacl/syscalls/irt_poll.h',
'$NACL-nacl/include/irt_poll.h'),
('bionic/libc/arch-nacl/syscalls/irt_socket.h',
'$NACL-nacl/include/irt_socket.h'),
('bionic/libc/include', '$NACL-nacl/include'),
('bionic/libc/arch-nacl/syscalls/nacl_stat.h',
'$NACL-nacl/include/nacl_stat.h'),
('bionic/libc/arch-$ARCH/include/machine',
'$NACL-nacl/include/machine'),
('bionic/libc/kernel/common', '$NACL-nacl/include'),
('bionic/libc/kernel/arch-$ARCH/asm', '$NACL-nacl/include/asm'),
('bionic/libm/include', '$NACL-nacl/include'),
('bionic/libm/$CPU', '$NACL-nacl/include'),
('bionic/safe-iop/include', '$NACL-nacl/include'),
('bionic/libstdc++/nacl', '$NACL-nacl/include/c++/4.8.2/$NACL-nacl'),
('bionic/nacl/$ARCH', '.'),
]
for arch in ARCHES:
for name in ['irt.h', 'irt_dev.h']:
src = os.path.join(NATIVE_CLIENT, 'src', 'untrusted', 'irt', name)
dst = GetToolchainPath(arch, 'bionic', '$NACL-nacl', 'include',
name)
MungeIRT(src, ReplaceArch(dst, arch))
inspath = GetToolchainPath(arch, 'bionic')
inspath = ReplaceArch(inspath, arch)
# Create empty objects and libraries
libpath = ReplaceArch(os.path.join(inspath, '$NACL-nacl', 'lib'), arch)
for name in EMPTY_FILES:
UpdateText(os.path.join(libpath, name), EMPTY)
# Copy BIONIC files to toolchain
for src, dst in BIONIC_PAIRS:
srcpath = ReplaceArch(os.path.join(TOOLCHAIN_BUILD_SRC, src), arch)
dstpath = ReplaceArch(os.path.join(inspath, dst), arch)
UpdateFromTo(srcpath, dstpath)
# Build specs file
gcc = ReplaceArch(os.path.join(inspath, 'bin', '$NACL-nacl-gcc'), arch)
lib = ReplaceArch(os.path.join(inspath, 'lib/gcc/$NACL-nacl/$VER'),
arch)
specs = os.path.join(lib, 'specs')
with open(specs, 'w') as specfile:
process.Run([gcc, '-dumpspecs'],
cwd=None,
shell=False,
outfile=specfile,
verbose=False)
text = open(specs, 'r').read()
# Replace items in the spec file
text = ReplaceText(text, [{
'-lgcc':
'-lgcc --as-needed %{!static: -lgcc_s} --no-as-needed %{!shared: -lgcc_eh}',
'--hash-style=gnu': '--hash-style=sysv',
}])
open(specs, 'w').write(text)
def CreateBasicToolchain():
# Create a toolchain directory containing only the toolchain binaries and
# basic files line nacl_arm_macros.s.
UpdateFromTo(
ARM_NEWLIB,
ARM_BIONIC,
filters=['*arm-nacl/include*', '*arm-nacl/lib*', '*.a', '*.o'])
UpdateFromTo(ARM_NEWLIB, ARM_BIONIC, paterns=['*.s'])
UpdateFromTo(X86_NEWLIB,
X86_BIONIC,
filters=['*x86_64-nacl/include*', '*x86_64-nacl/lib*', '*.o'])
UpdateFromTo(X86_NEWLIB, X86_BIONIC, paterns=['*.s'])
# Static build uses:
# crt1.o crti.o 4.8.2/crtbeginT.o ... 4.8.2/crtend.o crtn.o
# -shared build uses:
# crti.o 4.8.2/crtbeginS.o ... 4.8.2/crtendS.o crtn.o crtn.o
# However we only provide a crtbegin(S) and crtend(S)
EMPTY = """/*
* This is a dummy linker script.
* libnacl.a, libcrt_common.a, crt0.o crt1.o crti.o and crtn.o are all
* empty. Instead all the work is done by crtbegin(S).o and crtend(S).o and
* the bionic libc. These are provided for compatability with the newlib
* toolchain binaries.
*/"""
EMPTY_FILES = [
'crt0.o', 'crt1.o', 'crti.o', 'crtn.o', 'libnacl.a', 'libcrt_common.a',
'libpthread.a'
]
# Bionic uses the following include paths
BIONIC_PAIRS = [
('bionic/libc/include', '$NACL-nacl/include'),
('bionic/libc/arch-nacl/syscalls/irt.h', '$NACL-nacl/include/irt.h'),
('bionic/libc/arch-nacl/syscalls/irt_syscalls.h',
'$NACL-nacl/include/irt_syscalls.h'),
('bionic/libc/arch-nacl/syscalls/nacl_stat.h',
'$NACL-nacl/include/nacl_stat.h'),
('../../src/untrusted/irt/irt_dev.h', '$NACL-nacl/include/irt_dev.h'),
('bionic/libc/arch-$ARCH/include/machine',
'$NACL-nacl/include/machine'),
('bionic/libc/kernel/common', '$NACL-nacl/include'),
('bionic/libc/kernel/arch-$ARCH/asm', '$NACL-nacl/include/asm'),
('bionic/libm/include', '$NACL-nacl/include'),
('bionic/libm/$CPU', '$NACL-nacl/include'),
('bionic/safe-iop/include', '$NACL-nacl/include'),
('bionic/libstdc++/nacl', '$NACL-nacl/include/c++/4.8.2/$NACL-nacl'),
('bionic/nacl/$ARCH', '.'),
]
for arch in ARCHES:
inspath = os.path.join(TOOLCHAIN, 'linux_$ARCH_bionic')
inspath = ReplaceArch(inspath, arch)
# Create empty objects and libraries
libpath = ReplaceArch(os.path.join(inspath, '$NACL-nacl', 'lib'), arch)
for name in EMPTY_FILES:
UpdateText(os.path.join(libpath, name), EMPTY)
# Copy BIONIC files to toolchain
for src, dst in BIONIC_PAIRS:
srcpath = ReplaceArch(os.path.join(TOOLCHAIN_BUILD_SRC, src), arch)
dstpath = ReplaceArch(os.path.join(inspath, dst), arch)
UpdateFromTo(srcpath, dstpath)
workpath = os.path.join(TOOLCHAIN_BUILD_OUT, 'bionic_$ARCH_work')
workpath = ReplaceArch(workpath, arch)
ConfigureAndBuild(arch, 'bionic/libc', workpath, inspath)
ConfigureAndBuild(arch, 'bionic/libm', workpath, inspath)
# ConfigureAndBuild(arch, 'bionic/linker', workpath, inspath)
# Build specs file
gcc = ReplaceArch(os.path.join(inspath, 'bin', '$NACL-nacl-gcc'), arch)
lib = ReplaceArch(os.path.join(inspath, 'lib/gcc/$NACL-nacl/$VER'),
arch)
specs = os.path.join(lib, 'specs')
with open(specs, 'w') as specfile:
process.Run([gcc, '-dumpspecs'],
cwd=None,
shell=False,
outfile=specfile,
verbose=False)
text = open(specs, 'r').read()
text = ReplaceText(text, [{'-lgcc': '-lgcc %{!shared: -lgcc_eh}'}])
open(specs, 'w').write(text)
def main(argv):
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument(
'-v', '--verbose', dest='verbose',
default=False, action='store_true',
help='Produce more output.')
parser.add_argument(
'-c', '--clobber', dest='clobber',
default=False, action='store_true',
help='Clobber working directories before building.')
parser.add_argument(
'-f', '--fast-clobber', dest='fast_clobber',
default=False, action='store_true',
help='Clobber bionic working directories before building.')
parser.add_argument(
'-s', '--sync', dest='sync',
default=False, action='store_true',
help='Sync sources first.')
parser.add_argument(
'-b', '--buildbot', dest='buildbot',
default=False, action='store_true',
help='Running on the buildbot.')
parser.add_argument(
'-l', '--llvm', dest='llvm',
default=False, action='store_true',
help='Enable building via llvm.')
parser.add_argument(
'-u', '--upload', dest='upload',
default=False, action='store_true',
help='Upload build artifacts.')
parser.add_argument(
'--packages-file', dest='packages_file',
default=None,
help='Output packages file describing list of package files built.')
parser.add_argument(
'--skip-gcc', dest='skip_gcc',
default=False, action='store_true',
help='Skip building GCC components.')
options, leftover_args = parser.parse_known_args()
if '-h' in leftover_args or '--help' in leftover_args:
print 'The following arguments are specific to toolchain_build_bionic.py:'
parser.print_help()
print 'The rest of the arguments are generic, in toolchain_main.py'
return 1
if options.llvm:
ARCHES.append('pnacl')
if options.buildbot or options.upload:
version = os.environ['BUILDBOT_REVISION']
if options.clobber or options.fast_clobber:
Clobber(fast=options.fast_clobber)
if options.sync or options.buildbot:
FetchBionicSources()
# Copy headers and compiler tools
CreateBasicToolchain()
# Configure Bionic Projects, libc, libm, linker, tests, ...
ConfigureBionicProjects(clobber=options.buildbot)
# Build and install IRT header before building GCC
MakeBionicProject('libc', ['irt'])
if not options.skip_gcc:
# Build newlib gcc_libs for use by bionic
FetchAndBuild_gcc_libs()
# Configure and build libgcc.a
ConfigureAndBuild_libgcc(skip_build=options.skip_gcc)
# With libgcc.a, we can now build libc.so
MakeBionicProject('libc')
# With libc.so, we can build libgcc_s.so
BuildAndInstall_libgcc_s(skip_build=options.skip_gcc)
# With libc and libgcc_s, we can now build libm
MakeBionicProject('libm')
# With libc, libgcc, and libm, we can now build libstdc++
ConfigureAndBuild_libstdcpp()
# Now we can build the linker
#MakeBionicProject('linker')
MakeBionicProject('newlinker')
# Now we have a full toolchain, so test it
#MakeBionicProject('tests')
MakeBionicProject('newtests')
# We can run only off buildbots
if not options.buildbot:
process.Run(['./scons', 'platform=arm', '--mode=nacl,dbg-linux', '-j20'],
cwd=NATIVE_CLIENT)
MakeBionicProject('tests', ['run'])
MakeBionicProject('newtests', ['run'])
dst = os.path.join(TOOLCHAIN_BUILD_OUT, 'linux_arm_bionic', 'log.txt')
with open(dst, 'w') as dstf:
process.Run(['git', 'log', '-n', '1'],
cwd=os.path.join(TOOLCHAIN_BUILD_SRC, 'bionic'),
outfile=dstf,
verbose=False)
if options.buildbot or options.upload:
zipname = 'naclsdk_linux_arm_bionic.tgz'
ArchiveAndUpload(version, zipname, 'linux_arm_bionic',
options.packages_file)