def BuildStepArchiveSDKTools():
# Only push up sdk_tools.tgz and nacl_sdk.zip on the linux buildbot.
builder_name = os.getenv('BUILDBOT_BUILDERNAME', '')
if builder_name == 'linux-sdk-multi':
buildbot_common.BuildStep('Archive SDK Tools')
bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
build_utils.ChromeVersion(),)
buildbot_common.Archive('sdk_tools.tgz', bucket_path, OUT_DIR,
step_link=False)
buildbot_common.Archive('nacl_sdk.zip', bucket_path, OUT_DIR,
step_link=False)
def UploadArchives():
major_version = build_version.ChromeMajorVersion()
chrome_revision = build_version.ChromeRevision()
commit_position = build_version.ChromeCommitPosition()
git_sha = build_version.ParseCommitPosition(commit_position)[0]
short_sha = git_sha[:9]
archive_version = '%s-%s-%s' % (major_version, chrome_revision, short_sha)
bucket_path = 'native-client-sdk/archives/%s' % archive_version
for archive_name in all_archives:
buildbot_common.Archive(archive_name, bucket_path,
cwd=BUILD_ARCHIVE_DIR, step_link=False)
sha1_filename = archive_name + '.sha1'
buildbot_common.Archive(sha1_filename, bucket_path,
cwd=BUILD_ARCHIVE_DIR, step_link=False)
def Archive(filename, from_directory, step_link=True):
if buildbot_common.IsSDKBuilder():
bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/'
else:
bucket_path = 'nativeclient-mirror/nacl/nacl_sdk_test/'
bucket_path += build_version.ChromeVersion()
buildbot_common.Archive(filename, bucket_path, from_directory, step_link)
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument('--install-dir',
help='Install Directory',
dest='install_dir',
default='naclmono')
parser.add_argument('--tar-path',
help='Tarfile path',
dest='tar_path',
default='naclmono_%pepperrev%.bz2')
parser.add_argument(
'--upload-path',
help='Upload path (nativeclient-mirror/nacl/nacl_sdk/XXX)',
dest='upload_path',
default=None)
parser.add_argument('--pepper-revision',
help='Pepper revision',
dest='pepper_revision',
default=None)
parser.add_argument('--skip-upload',
help='Skips upload step',
action="store_true",
dest='skip_upload')
options = parser.parse_args(args)
if not options.upload_path:
buildbot_common.ErrorExit('--upload-path is required')
if not options.pepper_revision:
buildbot_common.ErrorExit('--pepper-revision is required')
options.tar_path = options.tar_path.replace('%pepperrev%',
options.pepper_revision)
install_folders = [
'bin', 'etc', 'include', 'lib', 'lib32', 'libarm', 'share'
]
buildbot_common.BuildStep('Archive Build')
tar_file = None
buildbot_common.RemoveFile(options.tar_path)
try:
tar_file = tarfile.open(options.tar_path,
mode='w:bz2',
dereference=True)
for subfolder in install_folders:
tar_file.add(os.path.join(options.install_dir, subfolder),
arcname=subfolder)
finally:
if tar_file:
tar_file.close()
if not options.skip_upload:
buildbot_common.Archive(
os.path.basename(options.tar_path),
'nativeclient-mirror/nacl/nacl_sdk/%s' % options.upload_path,
cwd=os.path.dirname(os.path.abspath(options.tar_path)))
def BuildStepArchiveBundle(pepper_ver, revision, tarfile):
buildbot_common.BuildStep('Archive build')
bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
build_utils.ChromeVersion(),)
tarname = os.path.basename(tarfile)
tarfile_dir = os.path.dirname(tarfile)
buildbot_common.Archive(tarname, bucket_path, tarfile_dir)
# generate "manifest snippet" for this archive.
archive_url = GSTORE + 'nacl_sdk/%s/%s' % (
build_utils.ChromeVersion(), tarname)
bundle = GetManifestBundle(pepper_ver, revision, tarfile, archive_url)
manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json')
with open(manifest_snippet_file, 'wb') as manifest_snippet_stream:
manifest_snippet_stream.write(bundle.GetDataAsString())
buildbot_common.Archive(tarname + '.json', bucket_path, OUT_DIR,
step_link=False)
def main(args):
parser = optparse.OptionParser()
parser.add_option('--pnacl',
help='Enable pnacl build.',
action='store_true',
dest='pnacl',
default=False)
parser.add_option('--examples',
help='Rebuild the examples.',
action='store_true',
dest='examples',
default=False)
parser.add_option('--update',
help='Rebuild the updater.',
action='store_true',
dest='update',
default=False)
parser.add_option('--skip-tar',
help='Skip generating a tarball.',
action='store_true',
dest='skip_tar',
default=False)
parser.add_option('--archive',
help='Force the archive step.',
action='store_true',
dest='archive',
default=False)
parser.add_option('--release',
help='PPAPI release version.',
dest='release',
default=None)
options, args = parser.parse_args(args[1:])
platform = getos.GetPlatform()
arch = 'x86'
builder_name = os.getenv('BUILDBOT_BUILDERNAME', '')
if builder_name.find('pnacl') >= 0 and builder_name.find('sdk') >= 0:
options.pnacl = True
if options.pnacl:
toolchains = ['pnacl']
else:
toolchains = ['newlib', 'glibc']
print 'Building: ' + ' '.join(toolchains)
skip = options.examples or options.update
skip_examples = skip
skip_update = skip
skip_untar = skip
skip_build = skip
skip_tar = skip or options.skip_tar
if options.examples: skip_examples = False
skip_update = not options.update
if options.archive and (options.examples or options.skip_tar):
parser.error('Incompatible arguments with archive.')
pepper_ver = str(int(build_utils.ChromeMajorVersion()))
clnumber = lastchange.FetchVersionInfo(None).revision
if options.release:
pepper_ver = options.release
print 'Building PEPPER %s at %s' % (pepper_ver, clnumber)
if not skip_build:
buildbot_common.BuildStep('Rerun hooks to get toolchains')
buildbot_common.Run(['gclient', 'runhooks'],
cwd=SRC_DIR,
shell=(platform == 'win'))
buildbot_common.BuildStep('Clean Pepper Dir')
pepperdir = os.path.join(SRC_DIR, 'out', 'pepper_' + pepper_ver)
if not skip_untar:
buildbot_common.RemoveDir(pepperdir)
buildbot_common.MakeDir(os.path.join(pepperdir, 'toolchain'))
buildbot_common.MakeDir(os.path.join(pepperdir, 'tools'))
buildbot_common.BuildStep('Add Text Files')
files = ['AUTHORS', 'COPYING', 'LICENSE', 'NOTICE', 'README']
files = [os.path.join(SDK_SRC_DIR, filename) for filename in files]
oshelpers.Copy(['-v'] + files + [pepperdir])
# Clean out the temporary toolchain untar directory
if not skip_untar:
UntarToolchains(pepperdir, platform, arch, toolchains)
if not skip_build:
BuildToolchains(pepperdir, platform, arch, pepper_ver, toolchains)
buildbot_common.BuildStep('Copy make OS helpers')
buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'),
os.path.join(pepperdir, 'tools'))
if platform == 'win':
buildbot_common.BuildStep('Add MAKE')
http_download.HttpDownload(
GSTORE + MAKE, os.path.join(pepperdir, 'tools', 'make.exe'))
if not skip_examples:
CopyExamples(pepperdir, toolchains)
if not skip_tar:
buildbot_common.BuildStep('Tar Pepper Bundle')
tarname = 'naclsdk_' + platform + '.bz2'
if 'pnacl' in toolchains:
tarname = 'p' + tarname
tarfile = os.path.join(OUT_DIR, tarname)
buildbot_common.Run([
sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile,
'pepper_' + pepper_ver
],
cwd=NACL_DIR)
# Archive on non-trybots.
if options.archive or '-sdk' in os.environ.get('BUILDBOT_BUILDERNAME', ''):
buildbot_common.BuildStep('Archive build')
buildbot_common.Archive(
tarname, 'nativeclient-mirror/nacl/nacl_sdk/%s' %
build_utils.ChromeVersion(), OUT_DIR)
if not skip_examples:
buildbot_common.BuildStep('Test Build Examples')
filelist = os.listdir(os.path.join(pepperdir, 'examples'))
for filenode in filelist:
dirnode = os.path.join(pepperdir, 'examples', filenode)
makefile = os.path.join(dirnode, 'Makefile')
if os.path.isfile(makefile):
print "\n\nMake: " + dirnode
buildbot_common.Run(['make', 'all', '-j8'],
cwd=os.path.abspath(dirnode),
shell=True)
# Build SDK Tools
if not skip_update:
BuildUpdater()
return 0
def main(args):
parser = optparse.OptionParser()
parser.add_option('--pnacl',
help='Enable pnacl build.',
action='store_true',
dest='pnacl',
default=False)
parser.add_option('--examples',
help='Only build the examples.',
action='store_true',
dest='only_examples',
default=False)
parser.add_option('--update',
help='Only build the updater.',
action='store_true',
dest='only_updater',
default=False)
parser.add_option('--skip-tar',
help='Skip generating a tarball.',
action='store_true',
dest='skip_tar',
default=False)
parser.add_option('--archive',
help='Force the archive step.',
action='store_true',
dest='archive',
default=False)
parser.add_option('--release',
help='PPAPI release version.',
dest='release',
default=None)
options, args = parser.parse_args(args[1:])
platform = getos.GetPlatform()
arch = 'x86'
builder_name = os.getenv('BUILDBOT_BUILDERNAME', '')
if builder_name.find('pnacl') >= 0 and builder_name.find('sdk') >= 0:
options.pnacl = True
if options.pnacl:
toolchains = ['pnacl']
else:
toolchains = ['newlib', 'glibc']
print 'Building: ' + ' '.join(toolchains)
skip = options.only_examples or options.only_updater
skip_examples = skip and not options.only_examples
skip_update = skip and not options.only_updater
skip_untar = skip
skip_build = skip
skip_test_updater = skip
skip_tar = skip or options.skip_tar
if options.archive and (options.only_examples or options.skip_tar):
parser.error('Incompatible arguments with archive.')
pepper_ver = str(int(build_utils.ChromeMajorVersion()))
pepper_old = str(int(build_utils.ChromeMajorVersion()) - 1)
clnumber = build_utils.ChromeRevision()
if options.release:
pepper_ver = options.release
print 'Building PEPPER %s at %s' % (pepper_ver, clnumber)
if not skip_build:
buildbot_common.BuildStep('Rerun hooks to get toolchains')
buildbot_common.Run(['gclient', 'runhooks'],
cwd=SRC_DIR,
shell=(platform == 'win'))
buildbot_common.BuildStep('Clean Pepper Dirs')
pepperdir = os.path.join(SRC_DIR, 'out', 'pepper_' + pepper_ver)
pepperold = os.path.join(SRC_DIR, 'out', 'pepper_' + pepper_old)
buildbot_common.RemoveDir(pepperold)
if not skip_untar:
buildbot_common.RemoveDir(pepperdir)
buildbot_common.MakeDir(os.path.join(pepperdir, 'libraries'))
buildbot_common.MakeDir(os.path.join(pepperdir, 'toolchain'))
buildbot_common.MakeDir(os.path.join(pepperdir, 'tools'))
else:
buildbot_common.MakeDir(pepperdir)
if not skip_build:
buildbot_common.BuildStep('Add Text Files')
files = ['AUTHORS', 'COPYING', 'LICENSE', 'NOTICE', 'README']
files = [os.path.join(SDK_SRC_DIR, filename) for filename in files]
oshelpers.Copy(['-v'] + files + [pepperdir])
# Clean out the temporary toolchain untar directory
if not skip_untar:
UntarToolchains(pepperdir, platform, arch, toolchains)
if not skip_build:
BuildToolchains(pepperdir, platform, arch, pepper_ver, toolchains)
InstallHeaders(os.path.join(pepperdir, 'libraries'), pepper_ver,
'libs')
if not skip_build:
buildbot_common.BuildStep('Copy make OS helpers')
buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'),
os.path.join(pepperdir, 'tools'))
if platform == 'win':
buildbot_common.BuildStep('Add MAKE')
http_download.HttpDownload(
GSTORE + MAKE, os.path.join(pepperdir, 'tools', 'make.exe'))
rename_list = [
'ncval_x86_32', 'ncval_x86_64', 'sel_ldr_x86_32',
'sel_ldr_x86_64'
]
tools = os.path.join(pepperdir, 'tools')
for name in rename_list:
src = os.path.join(pepperdir, 'tools', name)
dst = os.path.join(pepperdir, 'tools', name + '.exe')
buildbot_common.Move(src, dst)
if not skip_examples:
CopyExamples(pepperdir, toolchains)
tarname = 'naclsdk_' + platform + '.bz2'
if 'pnacl' in toolchains:
tarname = 'p' + tarname
tarfile = os.path.join(OUT_DIR, tarname)
if not skip_tar:
buildbot_common.BuildStep('Tar Pepper Bundle')
buildbot_common.Run([
sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile,
'pepper_' + pepper_ver
],
cwd=NACL_DIR)
# Run build tests
buildbot_common.BuildStep('Run build_tools tests')
buildbot_common.Run([
sys.executable,
os.path.join(SDK_SRC_DIR, 'build_tools', 'tests', 'test_all.py')
])
# build sdk update
if not skip_update:
build_updater.BuildUpdater(OUT_DIR)
# start local server sharing a manifest + the new bundle
if not skip_test_updater and not skip_tar:
buildbot_common.BuildStep('Move bundle to localserver dir')
buildbot_common.MakeDir(SERVER_DIR)
buildbot_common.Move(tarfile, SERVER_DIR)
tarfile = os.path.join(SERVER_DIR, tarname)
server = None
try:
buildbot_common.BuildStep('Run local server')
server = test_server.LocalHTTPServer(SERVER_DIR)
buildbot_common.BuildStep('Generate manifest')
with open(tarfile, 'rb') as tarfile_stream:
archive_sha1, archive_size = manifest_util.DownloadAndComputeHash(
tarfile_stream)
archive = manifest_util.Archive(manifest_util.GetHostOS())
archive.CopyFrom({
'url': server.GetURL(tarname),
'size': archive_size,
'checksum': {
'sha1': archive_sha1
}
})
bundle = manifest_util.Bundle('pepper_' + pepper_ver)
bundle.CopyFrom({
'revision':
int(clnumber),
'repath':
'pepper_' + pepper_ver,
'version':
int(pepper_ver),
'description':
'Chrome %s bundle, revision %s' % (pepper_ver, clnumber),
'stability':
'dev',
'recommended':
'no',
'archives': [archive]
})
manifest = manifest_util.SDKManifest()
manifest.SetBundle(bundle)
manifest_name = 'naclsdk_manifest2.json'
with open(os.path.join(SERVER_DIR, manifest_name), 'wb') as \
manifest_stream:
manifest_stream.write(manifest.GetDataAsString())
# use newly built sdk updater to pull this bundle
buildbot_common.BuildStep('Update from local server')
naclsdk_sh = os.path.join(OUT_DIR, 'nacl_sdk', 'naclsdk')
if platform == 'win':
naclsdk_sh += '.bat'
buildbot_common.Run([
naclsdk_sh, '-U',
server.GetURL(manifest_name), 'update', 'pepper_' + pepper_ver
])
# If we are testing examples, do it in the newly pulled directory.
pepperdir = os.path.join(OUT_DIR, 'nacl_sdk',
'pepper_' + pepper_ver)
# kill server
finally:
if server:
server.Shutdown()
# build examples.
if not skip_examples:
buildbot_common.BuildStep('Test Build Examples')
example_dir = os.path.join(pepperdir, 'examples')
makefile = os.path.join(example_dir, 'Makefile')
if os.path.isfile(makefile):
print "\n\nMake: " + example_dir
buildbot_common.Run(['make', '-j8'],
cwd=os.path.abspath(example_dir),
shell=True)
# Archive on non-trybots.
buildername = os.environ.get('BUILDBOT_BUILDERNAME', '')
if options.archive or '-sdk' in buildername:
buildbot_common.BuildStep('Archive build')
bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % \
build_utils.ChromeVersion()
buildbot_common.Archive(tarname, bucket_path, os.path.dirname(tarfile))
if not skip_update:
# Only push up sdk_tools.tgz on the linux buildbot.
if buildername == 'linux-sdk-multi':
sdk_tools = os.path.join(OUT_DIR, 'sdk_tools.tgz')
buildbot_common.Archive('sdk_tools.tgz',
bucket_path,
OUT_DIR,
step_link=False)
# generate "manifest snippet" for this archive.
if not skip_test_updater:
archive = bundle.GetArchive(manifest_util.GetHostOS())
archive.url = 'https://commondatastorage.googleapis.com/' \
'nativeclient-mirror/nacl/nacl_sdk/%s/%s' % (
build_utils.ChromeVersion(), tarname)
manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json')
with open(manifest_snippet_file, 'wb') as manifest_snippet_stream:
manifest_snippet_stream.write(bundle.GetDataAsString())
buildbot_common.Archive(tarname + '.json',
bucket_path,
OUT_DIR,
step_link=False)
return 0