def BuildStepTestUpdater(platform, pepper_ver, revision, tarfile):
tarname = os.path.basename(tarfile)
server = None
try:
buildbot_common.BuildStep('Run local server')
server = test_server.LocalHTTPServer(SERVER_DIR)
buildbot_common.BuildStep('Generate manifest')
bundle = GetManifestBundle(pepper_ver, revision, tarfile,
server.GetURL(tarname))
manifest = manifest_util.SDKManifest()
manifest.SetBundle(bundle)
manifest_name = 'naclsdk_manifest2.json'
with open(os.path.join(SERVER_DIR, manifest_name), 'wb') as \
manifest_stream:
manifest_stream.write(manifest.GetDataAsString())
# use newly built sdk updater to pull this bundle
buildbot_common.BuildStep('Update from local server')
naclsdk_sh = os.path.join(OUT_DIR, 'nacl_sdk', 'naclsdk')
if platform == 'win':
naclsdk_sh += '.bat'
buildbot_common.Run([naclsdk_sh, '-U',
server.GetURL(manifest_name), 'update', 'pepper_' + pepper_ver])
# Return the new pepper directory as the one inside the downloaded SDK.
return os.path.join(OUT_DIR, 'nacl_sdk', 'pepper_' + pepper_ver)
# kill server
finally:
if server:
server.Shutdown()
def List(options, argv, config):
'''Usage: %prog [options] list
Lists the available SDK bundles that are available for download.'''
def PrintBundles(bundles):
for bundle in bundles:
InfoPrint(' %s' % bundle.name)
for key, value in bundle.iteritems():
if key not in (manifest_util.ARCHIVES_KEY,
manifest_util.NAME_KEY):
InfoPrint(' %s: %s' % (key, value))
DebugPrint("Running List command with: %s, %s" % (options, argv))
parser = optparse.OptionParser(usage=List.__doc__)
(list_options, args) = parser.parse_args(argv)
manifest = LoadManifestFromURLs([options.manifest_url] +
config.GetSources())
InfoPrint('Available bundles:')
PrintBundles(manifest.GetBundles())
# Print the local information.
manifest_path = os.path.join(options.user_data_dir,
options.manifest_filename)
local_manifest = LoadFromFile(manifest_path, manifest_util.SDKManifest())
InfoPrint('\nCurrently installed bundles:')
PrintBundles(local_manifest.GetBundles())
def _UploadManifest(self, manifest):
"""Upload a serialized manifest_util.SDKManifest object.
Upload one copy to gs://<BUCKET_PATH>/naclsdk_manifest2.json, and a copy to
gs://<BUCKET_PATH>/manifest_backups/naclsdk_manifest2.<TIMESTAMP>.json.
Args:
manifest: The new manifest to upload.
"""
new_manifest_string = manifest.GetDataAsString()
online_manifest_string = self.online_manifest.GetDataAsString()
if self.delegate.dryrun:
logger.info(''.join(list(difflib.unified_diff(
online_manifest_string.splitlines(1),
new_manifest_string.splitlines(1)))))
return
else:
online_manifest = manifest_util.SDKManifest()
online_manifest.LoadDataFromString(online_manifest_string)
if online_manifest == manifest:
logger.info('New manifest doesn\'t differ from online manifest.'
'Skipping upload.')
return
timestamp_manifest_path = GS_MANIFEST_BACKUP_DIR + \
GetTimestampManifestName()
self.delegate.GsUtil_cp('-', timestamp_manifest_path,
stdin=manifest.GetDataAsString())
# copy from timestampped copy over the official manifest.
self.delegate.GsUtil_cp(timestamp_manifest_path, GS_SDK_MANIFEST)
def _ReadCacheManifest(self):
"""Read the manifest at nacl_sdk/sdk_cache."""
manifest_filename = os.path.join(self.cache_dir, MANIFEST_BASENAME)
manifest = manifest_util.SDKManifest()
with open(manifest_filename) as stream:
manifest.LoadDataFromString(stream.read())
return manifest
def GetRepoManifest(self):
"""See Delegate.GetRepoManifest"""
with open(REPO_MANIFEST, 'r') as sdk_stream:
sdk_json_string = sdk_stream.read()
manifest = manifest_util.SDKManifest()
manifest.LoadDataFromString(sdk_json_string, add_missing_info=True)
return manifest
def _LoadCacheManifest(self):
"""Read the manifest from nacl_sdk/sdk_cache.
This manifest should only contain the sdk_tools bundle.
"""
manifest_filename = os.path.join(self.cache_dir, MANIFEST_BASENAME)
self.manifest = manifest_util.SDKManifest()
self.manifest.LoadDataFromString(open(manifest_filename).read())
self.sdk_tools_bundle = self.manifest.GetBundle('sdk_tools')
def _GetSdkToolsBundleRevision(self):
"""Get the sdk_tools bundle revision.
We get this from the checked-in path; this is the same file that
build_updater uses to specify the current revision of sdk_tools."""
manifest_filename = os.path.join(BUILD_TOOLS_DIR, 'json',
'naclsdk_manifest0.json')
manifest = manifest_util.SDKManifest()
manifest.LoadDataFromString(open(manifest_filename, 'r').read())
return manifest.GetBundle('sdk_tools').revision
def LoadManifestFromURLs(urls):
'''Returns a manifest loaded from |urls|, merged into one manifest.'''
manifest = manifest_util.SDKManifest()
for url in urls:
try:
url_stream = UrlOpen(url)
except urllib2.URLError as e:
raise Error('Unable to open %s. [%s]' % (url, e))
manifest_stream = cStringIO.StringIO()
manifest_util.DownloadAndComputeHash(url_stream, manifest_stream)
temp_manifest = manifest_util.SDKManifest()
temp_manifest.LoadDataFromString(manifest_stream.getvalue())
manifest.MergeManifest(temp_manifest)
def BundleFilter(bundle):
# Only add this bundle if it's supported on this platform.
return bundle.GetHostOSArchive()
manifest.FilterBundles(BundleFilter)
return manifest
def LoadRemoteManifest(url):
manifest = manifest_util.SDKManifest()
url_stream = None
try:
manifest_stream = cStringIO.StringIO()
url_stream = download.UrlOpen(url)
download.DownloadAndComputeHash(url_stream, manifest_stream)
except urllib2.URLError as e:
raise Error('Unable to read remote manifest from URL "%s".\n %s' % (
url, e))
finally:
if url_stream:
url_stream.close()
try:
manifest.LoadDataFromString(manifest_stream.getvalue())
return manifest
except manifest_util.Error as e:
raise Error('Parsing remote manifest from URL "%s" failed.\n %s' % (
url, e,))
def LoadLocalManifest(raise_on_error=False):
path = os.path.join(USER_DATA_DIR, MANIFEST_FILENAME)
manifest = manifest_util.SDKManifest()
try:
try:
with open(path) as f:
manifest_string = f.read()
except IOError as e:
raise Error('Unable to read manifest from "%s".\n %s' % (path, e))
try:
manifest.LoadDataFromString(manifest_string)
except Exception as e:
raise Error('Parsing local manifest "%s" failed.\n %s' % (path, e))
except Error as e:
if raise_on_error:
raise
else:
logging.warn(str(e))
return manifest
def testSources(self):
"""The sources command should allow adding/listing/removing of sources.
When a source is added, it will provide an additional set of bundles."""
other_manifest = manifest_util.SDKManifest()
self._AddDummyBundle(other_manifest, 'naclmono_23')
with open(os.path.join(self.basedir, 'source.json'), 'w') as stream:
stream.write(other_manifest.GetDataAsString())
source_json_url = self.server.GetURL('source.json')
self._WriteManifest()
output = self._Run(['sources', '--list'])
self.assertTrue('No external sources installed.' in output)
output = self._Run(['sources', '--add', source_json_url])
output = self._Run(['sources', '--list'])
self.assertTrue(source_json_url in output)
# Should be able to get info about that bundle.
output = self._Run(['info', 'naclmono_23'])
self.assertTrue('Unknown bundle' not in output)
self._Run(['sources', '--remove', source_json_url])
output = self._Run(['sources', '--list'])
self.assertTrue('No external sources installed.' in output)
def _ReadUploadedManifest(self):
self.uploaded_manifest = manifest_util.SDKManifest()
self.uploaded_manifest.LoadDataFromString(
self.files['naclsdk_manifest2.json'])
def __init__(self, delegate):
self.delegate = delegate
self.versions_to_update = []
self.locked_bundles = []
self.online_manifest = manifest_util.SDKManifest()
self._FetchOnlineManifest()
def Update(options, argv, config):
'''Usage: %prog [options] update [target]
Updates the Native Client SDK to a specified version. By default, this
command updates all the recommended components. The update process works
like this:
1. Fetch the manifest from the mirror.
2. Load manifest from USER_DATA_DIR - if there is no local manifest file,
make an empty manifest object.
3. Update each the bundle:
for bundle in bundles:
# Compare bundle versions & revisions.
# Test if local version.revision < mirror OR local doesn't exist.
if local_manifest < mirror_manifest:
update(bundle)
update local_manifest with mirror_manifest for bundle
write manifest to disk. Use locks.
else:
InfoPrint('bundle is up-to-date')
Targets:
recommended: (default) Install/Update all recommended components
all: Install/Update all available components
bundle_name: Install/Update only the given bundle
'''
DebugPrint("Running Update command with: %s, %s" % (options, argv))
ALL = 'all' # Update all bundles
RECOMMENDED = 'recommended' # Only update the bundles with recommended=yes
parser = optparse.OptionParser(usage=Update.__doc__)
parser.add_option(
'-F',
'--force',
dest='force',
default=False,
action='store_true',
help='Force updating existing components that already exist')
(update_options, args) = parser.parse_args(argv)
if len(args) == 0:
args = [RECOMMENDED]
manifest = LoadManifestFromURLs([options.manifest_url] +
config.GetSources())
bundles = manifest.GetBundles()
local_manifest_path = os.path.join(options.user_data_dir,
options.manifest_filename)
local_manifest = LoadFromFile(local_manifest_path,
manifest_util.SDKManifest())
# Validate the arg list against the available bundle names. Raises an
# error if any invalid bundle names or args are detected.
valid_args = set([ALL, RECOMMENDED] + [bundle.name for bundle in bundles])
bad_args = set(args) - valid_args
if len(bad_args) > 0:
raise Error("Unrecognized bundle name or argument: '%s'" %
', '.join(bad_args))
for bundle in bundles:
bundle_path = os.path.join(options.sdk_root_dir, bundle.name)
bundle_update_path = '%s_update' % bundle_path
if bundle.name == SDK_TOOLS and not options.update_sdk_tools:
# We only want sdk_tools to updated by sdk_update.py. If the
# user tries to update directly, we just ignore the request.
InfoPrint('Updating sdk_tools happens automatically.\n'
'Ignoring manual update request.')
continue
if not (bundle.name in args or ALL in args or
(RECOMMENDED in args and bundle[RECOMMENDED] == 'yes')):
continue
def UpdateBundle():
'''Helper to install a bundle'''
archive = bundle.GetHostOSArchive()
(scheme, host, path, _, _, _) = urlparse.urlparse(archive['url'])
dest_filename = os.path.join(options.user_data_dir,
path.split('/')[-1])
sha1, size = DownloadArchiveToFile(archive, dest_filename)
if sha1 != archive.GetChecksum():
raise Error(
"SHA1 checksum mismatch on '%s'. Expected %s but got %s" %
(bundle.name, archive.GetChecksum(), sha1))
if size != archive.size:
raise Error(
"Size mismatch on Archive. Expected %s but got %s bytes" %
(archive.size, size))
InfoPrint('Updating bundle %s to version %s, revision %s' %
((bundle.name, bundle.version, bundle.revision)))
ExtractInstaller(dest_filename, bundle_update_path)
if bundle.name != SDK_TOOLS:
repath = bundle.get('repath', None)
if repath:
bundle_move_path = os.path.join(bundle_update_path, repath)
else:
bundle_move_path = bundle_update_path
RenameDir(bundle_move_path, bundle_path)
if os.path.exists(bundle_update_path):
RemoveDir(bundle_update_path)
os.remove(dest_filename)
local_manifest.MergeBundle(bundle)
WriteToFile(local_manifest_path, local_manifest)
# Test revision numbers, update the bundle accordingly.
# TODO(dspringer): The local file should be refreshed from disk each
# iteration thought this loop so that multiple sdk_updates can run at the
# same time.
if local_manifest.BundleNeedsUpdate(bundle):
if (not update_options.force and os.path.exists(bundle_path)
and bundle.name != SDK_TOOLS):
WarningPrint(
'%s already exists, but has an update available.\n'
'Run update with the --force option to overwrite the '
'existing directory.\nWarning: This will overwrite any '
'modifications you have made within this directory.' %
bundle.name)
else:
UpdateBundle()
else:
InfoPrint('%s is already up-to-date.' % bundle.name)
def main(args):
parser = optparse.OptionParser()
parser.add_option('--pnacl',
help='Enable pnacl build.',
action='store_true',
dest='pnacl',
default=False)
parser.add_option('--examples',
help='Only build the examples.',
action='store_true',
dest='only_examples',
default=False)
parser.add_option('--update',
help='Only build the updater.',
action='store_true',
dest='only_updater',
default=False)
parser.add_option('--skip-tar',
help='Skip generating a tarball.',
action='store_true',
dest='skip_tar',
default=False)
parser.add_option('--archive',
help='Force the archive step.',
action='store_true',
dest='archive',
default=False)
parser.add_option('--release',
help='PPAPI release version.',
dest='release',
default=None)
options, args = parser.parse_args(args[1:])
platform = getos.GetPlatform()
arch = 'x86'
builder_name = os.getenv('BUILDBOT_BUILDERNAME', '')
if builder_name.find('pnacl') >= 0 and builder_name.find('sdk') >= 0:
options.pnacl = True
if options.pnacl:
toolchains = ['pnacl']
else:
toolchains = ['newlib', 'glibc']
print 'Building: ' + ' '.join(toolchains)
skip = options.only_examples or options.only_updater
skip_examples = skip and not options.only_examples
skip_update = skip and not options.only_updater
skip_untar = skip
skip_build = skip
skip_test_updater = skip
skip_tar = skip or options.skip_tar
if options.archive and (options.only_examples or options.skip_tar):
parser.error('Incompatible arguments with archive.')
pepper_ver = str(int(build_utils.ChromeMajorVersion()))
pepper_old = str(int(build_utils.ChromeMajorVersion()) - 1)
clnumber = build_utils.ChromeRevision()
if options.release:
pepper_ver = options.release
print 'Building PEPPER %s at %s' % (pepper_ver, clnumber)
if not skip_build:
buildbot_common.BuildStep('Rerun hooks to get toolchains')
buildbot_common.Run(['gclient', 'runhooks'],
cwd=SRC_DIR,
shell=(platform == 'win'))
buildbot_common.BuildStep('Clean Pepper Dirs')
pepperdir = os.path.join(SRC_DIR, 'out', 'pepper_' + pepper_ver)
pepperold = os.path.join(SRC_DIR, 'out', 'pepper_' + pepper_old)
buildbot_common.RemoveDir(pepperold)
if not skip_untar:
buildbot_common.RemoveDir(pepperdir)
buildbot_common.MakeDir(os.path.join(pepperdir, 'libraries'))
buildbot_common.MakeDir(os.path.join(pepperdir, 'toolchain'))
buildbot_common.MakeDir(os.path.join(pepperdir, 'tools'))
else:
buildbot_common.MakeDir(pepperdir)
if not skip_build:
buildbot_common.BuildStep('Add Text Files')
files = ['AUTHORS', 'COPYING', 'LICENSE', 'NOTICE', 'README']
files = [os.path.join(SDK_SRC_DIR, filename) for filename in files]
oshelpers.Copy(['-v'] + files + [pepperdir])
# Clean out the temporary toolchain untar directory
if not skip_untar:
UntarToolchains(pepperdir, platform, arch, toolchains)
if not skip_build:
BuildToolchains(pepperdir, platform, arch, pepper_ver, toolchains)
InstallHeaders(os.path.join(pepperdir, 'libraries'), pepper_ver,
'libs')
if not skip_build:
buildbot_common.BuildStep('Copy make OS helpers')
buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'),
os.path.join(pepperdir, 'tools'))
if platform == 'win':
buildbot_common.BuildStep('Add MAKE')
http_download.HttpDownload(
GSTORE + MAKE, os.path.join(pepperdir, 'tools', 'make.exe'))
rename_list = [
'ncval_x86_32', 'ncval_x86_64', 'sel_ldr_x86_32',
'sel_ldr_x86_64'
]
tools = os.path.join(pepperdir, 'tools')
for name in rename_list:
src = os.path.join(pepperdir, 'tools', name)
dst = os.path.join(pepperdir, 'tools', name + '.exe')
buildbot_common.Move(src, dst)
if not skip_examples:
CopyExamples(pepperdir, toolchains)
tarname = 'naclsdk_' + platform + '.bz2'
if 'pnacl' in toolchains:
tarname = 'p' + tarname
tarfile = os.path.join(OUT_DIR, tarname)
if not skip_tar:
buildbot_common.BuildStep('Tar Pepper Bundle')
buildbot_common.Run([
sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile,
'pepper_' + pepper_ver
],
cwd=NACL_DIR)
# Run build tests
buildbot_common.BuildStep('Run build_tools tests')
buildbot_common.Run([
sys.executable,
os.path.join(SDK_SRC_DIR, 'build_tools', 'tests', 'test_all.py')
])
# build sdk update
if not skip_update:
build_updater.BuildUpdater(OUT_DIR)
# start local server sharing a manifest + the new bundle
if not skip_test_updater and not skip_tar:
buildbot_common.BuildStep('Move bundle to localserver dir')
buildbot_common.MakeDir(SERVER_DIR)
buildbot_common.Move(tarfile, SERVER_DIR)
tarfile = os.path.join(SERVER_DIR, tarname)
server = None
try:
buildbot_common.BuildStep('Run local server')
server = test_server.LocalHTTPServer(SERVER_DIR)
buildbot_common.BuildStep('Generate manifest')
with open(tarfile, 'rb') as tarfile_stream:
archive_sha1, archive_size = manifest_util.DownloadAndComputeHash(
tarfile_stream)
archive = manifest_util.Archive(manifest_util.GetHostOS())
archive.CopyFrom({
'url': server.GetURL(tarname),
'size': archive_size,
'checksum': {
'sha1': archive_sha1
}
})
bundle = manifest_util.Bundle('pepper_' + pepper_ver)
bundle.CopyFrom({
'revision':
int(clnumber),
'repath':
'pepper_' + pepper_ver,
'version':
int(pepper_ver),
'description':
'Chrome %s bundle, revision %s' % (pepper_ver, clnumber),
'stability':
'dev',
'recommended':
'no',
'archives': [archive]
})
manifest = manifest_util.SDKManifest()
manifest.SetBundle(bundle)
manifest_name = 'naclsdk_manifest2.json'
with open(os.path.join(SERVER_DIR, manifest_name), 'wb') as \
manifest_stream:
manifest_stream.write(manifest.GetDataAsString())
# use newly built sdk updater to pull this bundle
buildbot_common.BuildStep('Update from local server')
naclsdk_sh = os.path.join(OUT_DIR, 'nacl_sdk', 'naclsdk')
if platform == 'win':
naclsdk_sh += '.bat'
buildbot_common.Run([
naclsdk_sh, '-U',
server.GetURL(manifest_name), 'update', 'pepper_' + pepper_ver
])
# If we are testing examples, do it in the newly pulled directory.
pepperdir = os.path.join(OUT_DIR, 'nacl_sdk',
'pepper_' + pepper_ver)
# kill server
finally:
if server:
server.Shutdown()
# build examples.
if not skip_examples:
buildbot_common.BuildStep('Test Build Examples')
example_dir = os.path.join(pepperdir, 'examples')
makefile = os.path.join(example_dir, 'Makefile')
if os.path.isfile(makefile):
print "\n\nMake: " + example_dir
buildbot_common.Run(['make', '-j8'],
cwd=os.path.abspath(example_dir),
shell=True)
# Archive on non-trybots.
buildername = os.environ.get('BUILDBOT_BUILDERNAME', '')
if options.archive or '-sdk' in buildername:
buildbot_common.BuildStep('Archive build')
bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % \
build_utils.ChromeVersion()
buildbot_common.Archive(tarname, bucket_path, os.path.dirname(tarfile))
if not skip_update:
# Only push up sdk_tools.tgz on the linux buildbot.
if buildername == 'linux-sdk-multi':
sdk_tools = os.path.join(OUT_DIR, 'sdk_tools.tgz')
buildbot_common.Archive('sdk_tools.tgz',
bucket_path,
OUT_DIR,
step_link=False)
# generate "manifest snippet" for this archive.
if not skip_test_updater:
archive = bundle.GetArchive(manifest_util.GetHostOS())
archive.url = 'https://commondatastorage.googleapis.com/' \
'nativeclient-mirror/nacl/nacl_sdk/%s/%s' % (
build_utils.ChromeVersion(), tarname)
manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json')
with open(manifest_snippet_file, 'wb') as manifest_snippet_stream:
manifest_snippet_stream.write(bundle.GetDataAsString())
buildbot_common.Archive(tarname + '.json',
bucket_path,
OUT_DIR,
step_link=False)
return 0
def List(options, argv, config):
'''Usage: %prog [global_options] list [options]
Lists the available SDK bundles that are available for download.'''
def PrintBundle(local_bundle, bundle, needs_update, display_revisions):
installed = local_bundle is not None
# If bundle is None, there is no longer a remote bundle with this name.
if bundle is None:
bundle = local_bundle
if display_revisions:
if needs_update:
revision = ' (r%s -> r%s)' % (local_bundle.revision, bundle.revision)
else:
revision = ' (r%s)' % (bundle.revision,)
else:
revision = ''
InfoPrint(' %s%s %s (%s)%s' % (
'I' if installed else ' ',
'*' if needs_update else ' ',
bundle.name,
bundle.stability,
revision))
DebugPrint("Running List command with: %s, %s" %(options, argv))
parser = optparse.OptionParser(usage=List.__doc__)
parser.add_option(
'-r', '--revision', dest='revision',
default=False, action='store_true',
help='display revision numbers')
(list_options, _) = parser.parse_args(argv)
manifest = LoadManifestFromURLs([options.manifest_url] + config.GetSources())
manifest_path = os.path.join(options.user_data_dir, options.manifest_filename)
local_manifest = LoadFromFile(manifest_path, manifest_util.SDKManifest())
any_bundles_need_update = False
InfoPrint('Bundles:')
InfoPrint(' I: installed\n *: update available\n')
for bundle in manifest.GetBundles():
local_bundle = local_manifest.GetBundle(bundle.name)
needs_update = local_bundle and local_manifest.BundleNeedsUpdate(bundle)
if needs_update:
any_bundles_need_update = True
PrintBundle(local_bundle, bundle, needs_update, list_options.revision)
if not any_bundles_need_update:
InfoPrint('\nAll installed bundles are up-to-date.')
local_only_bundles = set([b.name for b in local_manifest.GetBundles()])
local_only_bundles -= set([b.name for b in manifest.GetBundles()])
if local_only_bundles:
InfoPrint('\nBundles installed locally that are not available remotely:')
for bundle_name in local_only_bundles:
local_bundle = local_manifest.GetBundle(bundle_name)
PrintBundle(local_bundle, None, False, list_options.revision)
