def ConfigureGCCProject(arch, project, cfg, workpath, inspath): # configure does not always have +x filepath = os.path.abspath(os.path.join(workpath, cfg[0])) st_info = os.stat(filepath) os.chmod(filepath, st_info.st_mode | stat.S_IEXEC) env = os.environ newpath = GetToolchainPath(arch, 'bionic', 'bin') + ':' + env['PATH'] proj = '%s %s' % (project, arch) setpath = ['/usr/bin/env', 'PATH=' + newpath] # Check if config changed or script is new config_path = os.path.join(workpath, 'config.info') updated = UpdateText(config_path, ' '.join(cfg)) updated |= NeedsUpdate(config_path, BUILD_SCRIPT) if updated: print 'Configure ' + proj if process.Run(setpath + cfg, cwd=workpath, env=env, outfile=sys.stdout): raise RuntimeError('Failed to configure %s.\n' % proj) else: print 'Reusing config for %s.' % proj
def FetchAndBuildGCC(): if 'arm' in ARCHES: tc_args = ['-y', '--no-use-remote-cache', 'gcc_libs_arm'] toolchain_main.PackageBuilder(toolchain_build.PACKAGES, tc_args).Main() if 'x86' in ARCHES: process.Run(['make', 'sync'], cwd=os.path.join(NATIVE_CLIENT, 'tools'))
def ArchiveAndUpload(version, zipname, zippath, packages_file): sys.stdout.flush() print >>sys.stderr, '@@@BUILD_STEP archive_and_upload@@@' bucket_path = 'nativeclient-archive2/toolchain/%s' % version gsd_store = pynacl.gsd_storage.GSDStorage(bucket_path, [bucket_path]) zipname = os.path.join(TOOLCHAIN_BUILD_OUT, zipname) try: os.remove(zipname) except: pass # Archive the zippath to the zipname. if process.Run(['tar', '-czf', zipname, zippath], cwd=TOOLCHAIN_BUILD_OUT, outfile=sys.stdout): raise RuntimeError('Failed to zip %s from %s.\n' % (zipname, zippath)) # Create Zip Hash file using the hash of the zip file. hashzipname = zipname + '.sha1hash' hashval = pynacl.hashing_tools.HashFileContents(zipname) with open(hashzipname, 'w') as f: f.write(hashval) # Upload the Zip file. zipurl = gsd_store.PutFile(zipname, os.path.basename(zipname)) sys.stdout.flush() print >>sys.stderr, ('@@@STEP_LINK@download (%s)@%s@@@' % (os.path.basename(zipname), zipurl)) # Upload the Zip Hash file. hashurl = gsd_store.PutFile(hashzipname, os.path.basename(hashzipname)) sys.stdout.flush() print >>sys.stderr, ('@@@STEP_LINK@download (%s)@%s@@@' % (os.path.basename(hashzipname), hashurl)) # Create a package info file for the nacl_arm_bionic package. archive_desc = archive_info.ArchiveInfo(name=os.path.basename(zipname), archive_hash=hashval, tar_src_dir='linux_arm_bionic', url=zipurl) package_desc = package_info.PackageInfo() package_desc.AppendArchive(archive_desc) os_name = pynacl.platform.GetOS() arch_name = pynacl.platform.GetArch() package_info_file = os.path.join(TOOLCHAIN_BUILD_OUT, 'packages', '%s_%s' % (os_name, arch_name), 'nacl_arm_bionic.json') package_desc.SavePackageFile(package_info_file) # If packages_file is specified, write out our packages file of 1 package. if packages_file: with open(packages_file, 'wt') as f: f.write(package_info_file)
def MakeBionicProject(project, targets=[], clobber=False): arch = 'arm' paths = GetProjectPaths(arch, project) workpath = paths['work'] inspath = paths['ins'] targetlist = ' '.join(targets) print 'Building %s for %s at %s %s.' % (project, arch, workpath, targetlist) if clobber: args = ['make', '-j12', 'V=1', 'clean'] if process.Run(args, cwd=workpath, outfile=sys.stdout): raise RuntimeError('Failed to clean %s for %s.\n' % (project, arch)) args = ['make', '-j12', 'V=1'] + targets if process.Run(args, cwd=workpath, outfile=sys.stdout): raise RuntimeError('Failed to build %s for %s.\n' % (project, arch)) print 'Done with %s for %s.\n' % (project, arch)
def ArchiveAndUpload(version, zipname, zippath): if 'BUILDBOT_BUILDERNAME' in os.environ: GSUTIL = '../buildbot/gsutil.sh' else: GSUTIL = 'gsutil' GSUTIL_ARGS = [GSUTIL, 'cp', '-a', 'public-read'] GSUTIL_PATH = 'gs://nativeclient-archive2/toolchain' urldir = os.path.join(GSUTIL_PATH, version) zipurl = os.path.join(urldir, zipname) zipname = os.path.join(TOOLCHAIN_BUILD_OUT, zipname) try: os.remove(zipname) except: pass sys.stdout.flush() print >> sys.stderr, '@@@STEP_LINK@download@%s@@@' % urldir if process.Run(['tar', '-czf', zipname, zippath], cwd=TOOLCHAIN, outfile=sys.stdout): raise RuntimeError('Failed to zip %s from %s.\n' % (zipname, zippath)) hashzipname = zipname + '.sha1hash' hashzipurl = zipurl + '.sha1hash' hashval = pynacl.hashing_tools.HashFileContents(zipname) with open(hashzipname, 'w') as f: f.write(hashval) if process.Run(GSUTIL_ARGS + [zipname, zipurl], cwd=TOOLCHAIN_BUILD, outfile=sys.stdout): err = 'Failed to upload zip %s to %s.\n' % (zipname, zipurl) raise RuntimeError(err) if process.Run(GSUTIL_ARGS + [hashzipname, hashzipurl], cwd=TOOLCHAIN_BUILD, outfile=sys.stdout): err = 'Failed to upload hash %s to %s.\n' % (hashzipname, hashzipurl) raise RuntimeError(err)
def ConfigureAndInstallForGCC(arch, project, cfg, workpath, inspath): # configure does not always have +x filepath = os.path.abspath(os.path.join(workpath, cfg[0])) st_info = os.stat(filepath) os.chmod(filepath, st_info.st_mode | stat.S_IEXEC) env = os.environ if arch == 'arm': newpath = os.path.join(ARM_BIONIC, 'bin') + ':' + env['PATH'] else: newpath = os.path.join(X86_BIONIC, 'bin') + ':' + env['PATH'] proj = '%s %s' % (project, arch) setpath = ['/usr/bin/env', 'PATH=' + newpath] # Check if config changed or script is new config_path = os.path.join(workpath, 'config.info') updated = UpdateText(config_path, ' '.join(cfg)) updated |= NeedsUpdate(config_path, BUILD_SCRIPT) if updated: print 'Configure ' + proj if process.Run(setpath + cfg, cwd=workpath, env=env, outfile=sys.stdout): raise RuntimeError('Failed to configure %s.\n' % proj) else: print 'Reusing config for %s.' % proj print 'Make ' + proj if process.Run(setpath + ['make', '-j16', 'V=1'], cwd=workpath, outfile=sys.stdout): raise RuntimeError('Failed to build %s.\n' % proj) print 'Install ' + proj if process.Run(setpath + ['make', '-j16', 'install', 'V=1'], cwd=workpath, outfile=sys.stdout): raise RuntimeError('Failed to install %s.\n' % proj) print 'Done ' + proj
def ConfigureAndBuild(arch, project, workpath, inspath, tcpath=None): # Create project for CRTx and LIBC files print 'Configure %s for %s.\n' % (project, arch) srcpath = os.path.join(TOOLCHAIN_BUILD_SRC, project) dstpath = ReplaceArch(os.path.join(workpath, '$NACL-nacl', project), arch) libpath = ReplaceArch(os.path.join(inspath, '$NACL-nacl', 'lib'), arch) CreateProject(arch, srcpath, dstpath, libpath) print 'Building %s for %s at %s.' % (project, arch, dstpath) if process.Run(['make', '-j12', 'V=1'], cwd=dstpath, outfile=sys.stdout): raise RuntimeError('Failed to build %s for %s.\n' % (project, arch)) if tcpath: UpdateFromTo(inspath, tcpath) print 'Done with %s for %s.\n' % (project, arch)
def MakeGCCProject(arch, project, workpath, targets=[]): env = os.environ newpath = GetBionicBuildPath(arch, 'bin') + ':' + env['PATH'] proj = '%s %s' % (project, arch) setpath = ['/usr/bin/env', 'PATH=' + newpath] if targets: proj = project = ': ' + ' '.join(targets) else: proj = project print 'Make ' + proj if process.Run(setpath + ['make', '-j16', 'V=1'] + targets, cwd=workpath, outfile=sys.stdout): raise RuntimeError('Failed to build %s.\n' % proj) print 'Done ' + proj
def CreateBasicToolchain(): # Create a toolchain directory containing only the toolchain binaries and # basic files line nacl_arm_macros.s. arch = 'arm' UpdateFromTo( GetToolchainPath(arch, 'newlib'), GetToolchainPath(arch, 'bionic'), filters=['*arm-nacl/include*', '*arm-nacl/lib*', '*.a', '*.o']) UpdateFromTo(GetToolchainPath(arch, 'newlib'), GetToolchainPath(arch, 'bionic'), paterns=['*.s']) # Static build uses: # crt1.o crti.o 4.8.2/crtbeginT.o ... 4.8.2/crtend.o crtn.o # -shared build uses: # crti.o 4.8.2/crtbeginS.o ... 4.8.2/crtendS.o crtn.o crtn.o # However we only provide a crtbegin(S) and crtend(S) EMPTY = """/* * This is a dummy linker script. * libnacl.a, libcrt_common.a, crt0.o crt1.o crti.o and crtn.o are all * empty. Instead all the work is done by crtbegin(S).o and crtend(S).o and * the bionic libc. These are provided for compatability with the newlib * toolchain binaries. */""" EMPTY_FILES = [ 'crt0.o', 'crt1.o', 'crti.o', 'crtn.o', 'libnacl.a', 'libcrt_common.a', 'libpthread.a' ] # Bionic uses the following include paths BIONIC_PAIRS = [ ('bionic/libc/arch-nacl/syscalls/irt_poll.h', '$NACL-nacl/include/irt_poll.h'), ('bionic/libc/arch-nacl/syscalls/irt_socket.h', '$NACL-nacl/include/irt_socket.h'), ('bionic/libc/include', '$NACL-nacl/include'), ('bionic/libc/arch-nacl/syscalls/nacl_stat.h', '$NACL-nacl/include/nacl_stat.h'), ('bionic/libc/arch-$ARCH/include/machine', '$NACL-nacl/include/machine'), ('bionic/libc/kernel/common', '$NACL-nacl/include'), ('bionic/libc/kernel/arch-$ARCH/asm', '$NACL-nacl/include/asm'), ('bionic/libm/include', '$NACL-nacl/include'), ('bionic/libm/$CPU', '$NACL-nacl/include'), ('bionic/safe-iop/include', '$NACL-nacl/include'), ('bionic/libstdc++/nacl', '$NACL-nacl/include/c++/4.8.2/$NACL-nacl'), ('bionic/nacl/$ARCH', '.'), ] for arch in ARCHES: for name in ['irt.h', 'irt_dev.h']: src = os.path.join(NATIVE_CLIENT, 'src', 'untrusted', 'irt', name) dst = GetToolchainPath(arch, 'bionic', '$NACL-nacl', 'include', name) MungeIRT(src, ReplaceArch(dst, arch)) inspath = GetToolchainPath(arch, 'bionic') inspath = ReplaceArch(inspath, arch) # Create empty objects and libraries libpath = ReplaceArch(os.path.join(inspath, '$NACL-nacl', 'lib'), arch) for name in EMPTY_FILES: UpdateText(os.path.join(libpath, name), EMPTY) # Copy BIONIC files to toolchain for src, dst in BIONIC_PAIRS: srcpath = ReplaceArch(os.path.join(TOOLCHAIN_BUILD_SRC, src), arch) dstpath = ReplaceArch(os.path.join(inspath, dst), arch) UpdateFromTo(srcpath, dstpath) # Build specs file gcc = ReplaceArch(os.path.join(inspath, 'bin', '$NACL-nacl-gcc'), arch) lib = ReplaceArch(os.path.join(inspath, 'lib/gcc/$NACL-nacl/$VER'), arch) specs = os.path.join(lib, 'specs') with open(specs, 'w') as specfile: process.Run([gcc, '-dumpspecs'], cwd=None, shell=False, outfile=specfile, verbose=False) text = open(specs, 'r').read() # Replace items in the spec file text = ReplaceText(text, [{ '-lgcc': '-lgcc --as-needed %{!static: -lgcc_s} --no-as-needed %{!shared: -lgcc_eh}', '--hash-style=gnu': '--hash-style=sysv', }]) open(specs, 'w').write(text)
def CreateBasicToolchain(): # Create a toolchain directory containing only the toolchain binaries and # basic files line nacl_arm_macros.s. UpdateFromTo( ARM_NEWLIB, ARM_BIONIC, filters=['*arm-nacl/include*', '*arm-nacl/lib*', '*.a', '*.o']) UpdateFromTo(ARM_NEWLIB, ARM_BIONIC, paterns=['*.s']) UpdateFromTo(X86_NEWLIB, X86_BIONIC, filters=['*x86_64-nacl/include*', '*x86_64-nacl/lib*', '*.o']) UpdateFromTo(X86_NEWLIB, X86_BIONIC, paterns=['*.s']) # Static build uses: # crt1.o crti.o 4.8.2/crtbeginT.o ... 4.8.2/crtend.o crtn.o # -shared build uses: # crti.o 4.8.2/crtbeginS.o ... 4.8.2/crtendS.o crtn.o crtn.o # However we only provide a crtbegin(S) and crtend(S) EMPTY = """/* * This is a dummy linker script. * libnacl.a, libcrt_common.a, crt0.o crt1.o crti.o and crtn.o are all * empty. Instead all the work is done by crtbegin(S).o and crtend(S).o and * the bionic libc. These are provided for compatability with the newlib * toolchain binaries. */""" EMPTY_FILES = [ 'crt0.o', 'crt1.o', 'crti.o', 'crtn.o', 'libnacl.a', 'libcrt_common.a', 'libpthread.a' ] # Bionic uses the following include paths BIONIC_PAIRS = [ ('bionic/libc/include', '$NACL-nacl/include'), ('bionic/libc/arch-nacl/syscalls/irt.h', '$NACL-nacl/include/irt.h'), ('bionic/libc/arch-nacl/syscalls/irt_syscalls.h', '$NACL-nacl/include/irt_syscalls.h'), ('bionic/libc/arch-nacl/syscalls/nacl_stat.h', '$NACL-nacl/include/nacl_stat.h'), ('../../src/untrusted/irt/irt_dev.h', '$NACL-nacl/include/irt_dev.h'), ('bionic/libc/arch-$ARCH/include/machine', '$NACL-nacl/include/machine'), ('bionic/libc/kernel/common', '$NACL-nacl/include'), ('bionic/libc/kernel/arch-$ARCH/asm', '$NACL-nacl/include/asm'), ('bionic/libm/include', '$NACL-nacl/include'), ('bionic/libm/$CPU', '$NACL-nacl/include'), ('bionic/safe-iop/include', '$NACL-nacl/include'), ('bionic/libstdc++/nacl', '$NACL-nacl/include/c++/4.8.2/$NACL-nacl'), ('bionic/nacl/$ARCH', '.'), ] for arch in ARCHES: inspath = os.path.join(TOOLCHAIN, 'linux_$ARCH_bionic') inspath = ReplaceArch(inspath, arch) # Create empty objects and libraries libpath = ReplaceArch(os.path.join(inspath, '$NACL-nacl', 'lib'), arch) for name in EMPTY_FILES: UpdateText(os.path.join(libpath, name), EMPTY) # Copy BIONIC files to toolchain for src, dst in BIONIC_PAIRS: srcpath = ReplaceArch(os.path.join(TOOLCHAIN_BUILD_SRC, src), arch) dstpath = ReplaceArch(os.path.join(inspath, dst), arch) UpdateFromTo(srcpath, dstpath) workpath = os.path.join(TOOLCHAIN_BUILD_OUT, 'bionic_$ARCH_work') workpath = ReplaceArch(workpath, arch) ConfigureAndBuild(arch, 'bionic/libc', workpath, inspath) ConfigureAndBuild(arch, 'bionic/libm', workpath, inspath) # ConfigureAndBuild(arch, 'bionic/linker', workpath, inspath) # Build specs file gcc = ReplaceArch(os.path.join(inspath, 'bin', '$NACL-nacl-gcc'), arch) lib = ReplaceArch(os.path.join(inspath, 'lib/gcc/$NACL-nacl/$VER'), arch) specs = os.path.join(lib, 'specs') with open(specs, 'w') as specfile: process.Run([gcc, '-dumpspecs'], cwd=None, shell=False, outfile=specfile, verbose=False) text = open(specs, 'r').read() text = ReplaceText(text, [{'-lgcc': '-lgcc %{!shared: -lgcc_eh}'}]) open(specs, 'w').write(text)
def main(argv): parser = argparse.ArgumentParser(add_help=False) parser.add_argument( '-v', '--verbose', dest='verbose', default=False, action='store_true', help='Produce more output.') parser.add_argument( '-c', '--clobber', dest='clobber', default=False, action='store_true', help='Clobber working directories before building.') parser.add_argument( '-f', '--fast-clobber', dest='fast_clobber', default=False, action='store_true', help='Clobber bionic working directories before building.') parser.add_argument( '-s', '--sync', dest='sync', default=False, action='store_true', help='Sync sources first.') parser.add_argument( '-b', '--buildbot', dest='buildbot', default=False, action='store_true', help='Running on the buildbot.') parser.add_argument( '-l', '--llvm', dest='llvm', default=False, action='store_true', help='Enable building via llvm.') parser.add_argument( '-u', '--upload', dest='upload', default=False, action='store_true', help='Upload build artifacts.') parser.add_argument( '--packages-file', dest='packages_file', default=None, help='Output packages file describing list of package files built.') parser.add_argument( '--skip-gcc', dest='skip_gcc', default=False, action='store_true', help='Skip building GCC components.') options, leftover_args = parser.parse_known_args() if '-h' in leftover_args or '--help' in leftover_args: print 'The following arguments are specific to toolchain_build_bionic.py:' parser.print_help() print 'The rest of the arguments are generic, in toolchain_main.py' return 1 if options.llvm: ARCHES.append('pnacl') if options.buildbot or options.upload: version = os.environ['BUILDBOT_REVISION'] if options.clobber or options.fast_clobber: Clobber(fast=options.fast_clobber) if options.sync or options.buildbot: FetchBionicSources() # Copy headers and compiler tools CreateBasicToolchain() # Configure Bionic Projects, libc, libm, linker, tests, ... ConfigureBionicProjects(clobber=options.buildbot) # Build and install IRT header before building GCC MakeBionicProject('libc', ['irt']) if not options.skip_gcc: # Build newlib gcc_libs for use by bionic FetchAndBuild_gcc_libs() # Configure and build libgcc.a ConfigureAndBuild_libgcc(skip_build=options.skip_gcc) # With libgcc.a, we can now build libc.so MakeBionicProject('libc') # With libc.so, we can build libgcc_s.so BuildAndInstall_libgcc_s(skip_build=options.skip_gcc) # With libc and libgcc_s, we can now build libm MakeBionicProject('libm') # With libc, libgcc, and libm, we can now build libstdc++ ConfigureAndBuild_libstdcpp() # Now we can build the linker #MakeBionicProject('linker') MakeBionicProject('newlinker') # Now we have a full toolchain, so test it #MakeBionicProject('tests') MakeBionicProject('newtests') # We can run only off buildbots if not options.buildbot: process.Run(['./scons', 'platform=arm', '--mode=nacl,dbg-linux', '-j20'], cwd=NATIVE_CLIENT) MakeBionicProject('tests', ['run']) MakeBionicProject('newtests', ['run']) dst = os.path.join(TOOLCHAIN_BUILD_OUT, 'linux_arm_bionic', 'log.txt') with open(dst, 'w') as dstf: process.Run(['git', 'log', '-n', '1'], cwd=os.path.join(TOOLCHAIN_BUILD_SRC, 'bionic'), outfile=dstf, verbose=False) if options.buildbot or options.upload: zipname = 'naclsdk_linux_arm_bionic.tgz' ArchiveAndUpload(version, zipname, 'linux_arm_bionic', options.packages_file)