def BuildStepTestUpdater(platform, pepper_ver, revision, tarfile): tarname = os.path.basename(tarfile) server = None try: buildbot_common.BuildStep('Run local server') server = test_server.LocalHTTPServer(SERVER_DIR) buildbot_common.BuildStep('Generate manifest') bundle = GetManifestBundle(pepper_ver, revision, tarfile, server.GetURL(tarname)) manifest = manifest_util.SDKManifest() manifest.SetBundle(bundle) manifest_name = 'naclsdk_manifest2.json' with open(os.path.join(SERVER_DIR, manifest_name), 'wb') as \ manifest_stream: manifest_stream.write(manifest.GetDataAsString()) # use newly built sdk updater to pull this bundle buildbot_common.BuildStep('Update from local server') naclsdk_sh = os.path.join(OUT_DIR, 'nacl_sdk', 'naclsdk') if platform == 'win': naclsdk_sh += '.bat' buildbot_common.Run([naclsdk_sh, '-U', server.GetURL(manifest_name), 'update', 'pepper_' + pepper_ver]) # Return the new pepper directory as the one inside the downloaded SDK. return os.path.join(OUT_DIR, 'nacl_sdk', 'pepper_' + pepper_ver) # kill server finally: if server: server.Shutdown()
def List(options, argv, config): '''Usage: %prog [options] list Lists the available SDK bundles that are available for download.''' def PrintBundles(bundles): for bundle in bundles: InfoPrint(' %s' % bundle.name) for key, value in bundle.iteritems(): if key not in (manifest_util.ARCHIVES_KEY, manifest_util.NAME_KEY): InfoPrint(' %s: %s' % (key, value)) DebugPrint("Running List command with: %s, %s" % (options, argv)) parser = optparse.OptionParser(usage=List.__doc__) (list_options, args) = parser.parse_args(argv) manifest = LoadManifestFromURLs([options.manifest_url] + config.GetSources()) InfoPrint('Available bundles:') PrintBundles(manifest.GetBundles()) # Print the local information. manifest_path = os.path.join(options.user_data_dir, options.manifest_filename) local_manifest = LoadFromFile(manifest_path, manifest_util.SDKManifest()) InfoPrint('\nCurrently installed bundles:') PrintBundles(local_manifest.GetBundles())
def _UploadManifest(self, manifest): """Upload a serialized manifest_util.SDKManifest object. Upload one copy to gs://<BUCKET_PATH>/naclsdk_manifest2.json, and a copy to gs://<BUCKET_PATH>/manifest_backups/naclsdk_manifest2.<TIMESTAMP>.json. Args: manifest: The new manifest to upload. """ new_manifest_string = manifest.GetDataAsString() online_manifest_string = self.online_manifest.GetDataAsString() if self.delegate.dryrun: logger.info(''.join(list(difflib.unified_diff( online_manifest_string.splitlines(1), new_manifest_string.splitlines(1))))) return else: online_manifest = manifest_util.SDKManifest() online_manifest.LoadDataFromString(online_manifest_string) if online_manifest == manifest: logger.info('New manifest doesn\'t differ from online manifest.' 'Skipping upload.') return timestamp_manifest_path = GS_MANIFEST_BACKUP_DIR + \ GetTimestampManifestName() self.delegate.GsUtil_cp('-', timestamp_manifest_path, stdin=manifest.GetDataAsString()) # copy from timestampped copy over the official manifest. self.delegate.GsUtil_cp(timestamp_manifest_path, GS_SDK_MANIFEST)
def _ReadCacheManifest(self): """Read the manifest at nacl_sdk/sdk_cache.""" manifest_filename = os.path.join(self.cache_dir, MANIFEST_BASENAME) manifest = manifest_util.SDKManifest() with open(manifest_filename) as stream: manifest.LoadDataFromString(stream.read()) return manifest
def GetRepoManifest(self): """See Delegate.GetRepoManifest""" with open(REPO_MANIFEST, 'r') as sdk_stream: sdk_json_string = sdk_stream.read() manifest = manifest_util.SDKManifest() manifest.LoadDataFromString(sdk_json_string, add_missing_info=True) return manifest
def _LoadCacheManifest(self): """Read the manifest from nacl_sdk/sdk_cache. This manifest should only contain the sdk_tools bundle. """ manifest_filename = os.path.join(self.cache_dir, MANIFEST_BASENAME) self.manifest = manifest_util.SDKManifest() self.manifest.LoadDataFromString(open(manifest_filename).read()) self.sdk_tools_bundle = self.manifest.GetBundle('sdk_tools')
def _GetSdkToolsBundleRevision(self): """Get the sdk_tools bundle revision. We get this from the checked-in path; this is the same file that build_updater uses to specify the current revision of sdk_tools.""" manifest_filename = os.path.join(BUILD_TOOLS_DIR, 'json', 'naclsdk_manifest0.json') manifest = manifest_util.SDKManifest() manifest.LoadDataFromString(open(manifest_filename, 'r').read()) return manifest.GetBundle('sdk_tools').revision
def LoadManifestFromURLs(urls): '''Returns a manifest loaded from |urls|, merged into one manifest.''' manifest = manifest_util.SDKManifest() for url in urls: try: url_stream = UrlOpen(url) except urllib2.URLError as e: raise Error('Unable to open %s. [%s]' % (url, e)) manifest_stream = cStringIO.StringIO() manifest_util.DownloadAndComputeHash(url_stream, manifest_stream) temp_manifest = manifest_util.SDKManifest() temp_manifest.LoadDataFromString(manifest_stream.getvalue()) manifest.MergeManifest(temp_manifest) def BundleFilter(bundle): # Only add this bundle if it's supported on this platform. return bundle.GetHostOSArchive() manifest.FilterBundles(BundleFilter) return manifest
def LoadRemoteManifest(url): manifest = manifest_util.SDKManifest() url_stream = None try: manifest_stream = cStringIO.StringIO() url_stream = download.UrlOpen(url) download.DownloadAndComputeHash(url_stream, manifest_stream) except urllib2.URLError as e: raise Error('Unable to read remote manifest from URL "%s".\n %s' % ( url, e)) finally: if url_stream: url_stream.close() try: manifest.LoadDataFromString(manifest_stream.getvalue()) return manifest except manifest_util.Error as e: raise Error('Parsing remote manifest from URL "%s" failed.\n %s' % ( url, e,))
def LoadLocalManifest(raise_on_error=False): path = os.path.join(USER_DATA_DIR, MANIFEST_FILENAME) manifest = manifest_util.SDKManifest() try: try: with open(path) as f: manifest_string = f.read() except IOError as e: raise Error('Unable to read manifest from "%s".\n %s' % (path, e)) try: manifest.LoadDataFromString(manifest_string) except Exception as e: raise Error('Parsing local manifest "%s" failed.\n %s' % (path, e)) except Error as e: if raise_on_error: raise else: logging.warn(str(e)) return manifest
def testSources(self): """The sources command should allow adding/listing/removing of sources. When a source is added, it will provide an additional set of bundles.""" other_manifest = manifest_util.SDKManifest() self._AddDummyBundle(other_manifest, 'naclmono_23') with open(os.path.join(self.basedir, 'source.json'), 'w') as stream: stream.write(other_manifest.GetDataAsString()) source_json_url = self.server.GetURL('source.json') self._WriteManifest() output = self._Run(['sources', '--list']) self.assertTrue('No external sources installed.' in output) output = self._Run(['sources', '--add', source_json_url]) output = self._Run(['sources', '--list']) self.assertTrue(source_json_url in output) # Should be able to get info about that bundle. output = self._Run(['info', 'naclmono_23']) self.assertTrue('Unknown bundle' not in output) self._Run(['sources', '--remove', source_json_url]) output = self._Run(['sources', '--list']) self.assertTrue('No external sources installed.' in output)
def _ReadUploadedManifest(self): self.uploaded_manifest = manifest_util.SDKManifest() self.uploaded_manifest.LoadDataFromString( self.files['naclsdk_manifest2.json'])
def __init__(self, delegate): self.delegate = delegate self.versions_to_update = [] self.locked_bundles = [] self.online_manifest = manifest_util.SDKManifest() self._FetchOnlineManifest()
def Update(options, argv, config): '''Usage: %prog [options] update [target] Updates the Native Client SDK to a specified version. By default, this command updates all the recommended components. The update process works like this: 1. Fetch the manifest from the mirror. 2. Load manifest from USER_DATA_DIR - if there is no local manifest file, make an empty manifest object. 3. Update each the bundle: for bundle in bundles: # Compare bundle versions & revisions. # Test if local version.revision < mirror OR local doesn't exist. if local_manifest < mirror_manifest: update(bundle) update local_manifest with mirror_manifest for bundle write manifest to disk. Use locks. else: InfoPrint('bundle is up-to-date') Targets: recommended: (default) Install/Update all recommended components all: Install/Update all available components bundle_name: Install/Update only the given bundle ''' DebugPrint("Running Update command with: %s, %s" % (options, argv)) ALL = 'all' # Update all bundles RECOMMENDED = 'recommended' # Only update the bundles with recommended=yes parser = optparse.OptionParser(usage=Update.__doc__) parser.add_option( '-F', '--force', dest='force', default=False, action='store_true', help='Force updating existing components that already exist') (update_options, args) = parser.parse_args(argv) if len(args) == 0: args = [RECOMMENDED] manifest = LoadManifestFromURLs([options.manifest_url] + config.GetSources()) bundles = manifest.GetBundles() local_manifest_path = os.path.join(options.user_data_dir, options.manifest_filename) local_manifest = LoadFromFile(local_manifest_path, manifest_util.SDKManifest()) # Validate the arg list against the available bundle names. Raises an # error if any invalid bundle names or args are detected. valid_args = set([ALL, RECOMMENDED] + [bundle.name for bundle in bundles]) bad_args = set(args) - valid_args if len(bad_args) > 0: raise Error("Unrecognized bundle name or argument: '%s'" % ', '.join(bad_args)) for bundle in bundles: bundle_path = os.path.join(options.sdk_root_dir, bundle.name) bundle_update_path = '%s_update' % bundle_path if bundle.name == SDK_TOOLS and not options.update_sdk_tools: # We only want sdk_tools to updated by sdk_update.py. If the # user tries to update directly, we just ignore the request. InfoPrint('Updating sdk_tools happens automatically.\n' 'Ignoring manual update request.') continue if not (bundle.name in args or ALL in args or (RECOMMENDED in args and bundle[RECOMMENDED] == 'yes')): continue def UpdateBundle(): '''Helper to install a bundle''' archive = bundle.GetHostOSArchive() (scheme, host, path, _, _, _) = urlparse.urlparse(archive['url']) dest_filename = os.path.join(options.user_data_dir, path.split('/')[-1]) sha1, size = DownloadArchiveToFile(archive, dest_filename) if sha1 != archive.GetChecksum(): raise Error( "SHA1 checksum mismatch on '%s'. Expected %s but got %s" % (bundle.name, archive.GetChecksum(), sha1)) if size != archive.size: raise Error( "Size mismatch on Archive. Expected %s but got %s bytes" % (archive.size, size)) InfoPrint('Updating bundle %s to version %s, revision %s' % ((bundle.name, bundle.version, bundle.revision))) ExtractInstaller(dest_filename, bundle_update_path) if bundle.name != SDK_TOOLS: repath = bundle.get('repath', None) if repath: bundle_move_path = os.path.join(bundle_update_path, repath) else: bundle_move_path = bundle_update_path RenameDir(bundle_move_path, bundle_path) if os.path.exists(bundle_update_path): RemoveDir(bundle_update_path) os.remove(dest_filename) local_manifest.MergeBundle(bundle) WriteToFile(local_manifest_path, local_manifest) # Test revision numbers, update the bundle accordingly. # TODO(dspringer): The local file should be refreshed from disk each # iteration thought this loop so that multiple sdk_updates can run at the # same time. if local_manifest.BundleNeedsUpdate(bundle): if (not update_options.force and os.path.exists(bundle_path) and bundle.name != SDK_TOOLS): WarningPrint( '%s already exists, but has an update available.\n' 'Run update with the --force option to overwrite the ' 'existing directory.\nWarning: This will overwrite any ' 'modifications you have made within this directory.' % bundle.name) else: UpdateBundle() else: InfoPrint('%s is already up-to-date.' % bundle.name)
def main(args): parser = optparse.OptionParser() parser.add_option('--pnacl', help='Enable pnacl build.', action='store_true', dest='pnacl', default=False) parser.add_option('--examples', help='Only build the examples.', action='store_true', dest='only_examples', default=False) parser.add_option('--update', help='Only build the updater.', action='store_true', dest='only_updater', default=False) parser.add_option('--skip-tar', help='Skip generating a tarball.', action='store_true', dest='skip_tar', default=False) parser.add_option('--archive', help='Force the archive step.', action='store_true', dest='archive', default=False) parser.add_option('--release', help='PPAPI release version.', dest='release', default=None) options, args = parser.parse_args(args[1:]) platform = getos.GetPlatform() arch = 'x86' builder_name = os.getenv('BUILDBOT_BUILDERNAME', '') if builder_name.find('pnacl') >= 0 and builder_name.find('sdk') >= 0: options.pnacl = True if options.pnacl: toolchains = ['pnacl'] else: toolchains = ['newlib', 'glibc'] print 'Building: ' + ' '.join(toolchains) skip = options.only_examples or options.only_updater skip_examples = skip and not options.only_examples skip_update = skip and not options.only_updater skip_untar = skip skip_build = skip skip_test_updater = skip skip_tar = skip or options.skip_tar if options.archive and (options.only_examples or options.skip_tar): parser.error('Incompatible arguments with archive.') pepper_ver = str(int(build_utils.ChromeMajorVersion())) pepper_old = str(int(build_utils.ChromeMajorVersion()) - 1) clnumber = build_utils.ChromeRevision() if options.release: pepper_ver = options.release print 'Building PEPPER %s at %s' % (pepper_ver, clnumber) if not skip_build: buildbot_common.BuildStep('Rerun hooks to get toolchains') buildbot_common.Run(['gclient', 'runhooks'], cwd=SRC_DIR, shell=(platform == 'win')) buildbot_common.BuildStep('Clean Pepper Dirs') pepperdir = os.path.join(SRC_DIR, 'out', 'pepper_' + pepper_ver) pepperold = os.path.join(SRC_DIR, 'out', 'pepper_' + pepper_old) buildbot_common.RemoveDir(pepperold) if not skip_untar: buildbot_common.RemoveDir(pepperdir) buildbot_common.MakeDir(os.path.join(pepperdir, 'libraries')) buildbot_common.MakeDir(os.path.join(pepperdir, 'toolchain')) buildbot_common.MakeDir(os.path.join(pepperdir, 'tools')) else: buildbot_common.MakeDir(pepperdir) if not skip_build: buildbot_common.BuildStep('Add Text Files') files = ['AUTHORS', 'COPYING', 'LICENSE', 'NOTICE', 'README'] files = [os.path.join(SDK_SRC_DIR, filename) for filename in files] oshelpers.Copy(['-v'] + files + [pepperdir]) # Clean out the temporary toolchain untar directory if not skip_untar: UntarToolchains(pepperdir, platform, arch, toolchains) if not skip_build: BuildToolchains(pepperdir, platform, arch, pepper_ver, toolchains) InstallHeaders(os.path.join(pepperdir, 'libraries'), pepper_ver, 'libs') if not skip_build: buildbot_common.BuildStep('Copy make OS helpers') buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'), os.path.join(pepperdir, 'tools')) if platform == 'win': buildbot_common.BuildStep('Add MAKE') http_download.HttpDownload( GSTORE + MAKE, os.path.join(pepperdir, 'tools', 'make.exe')) rename_list = [ 'ncval_x86_32', 'ncval_x86_64', 'sel_ldr_x86_32', 'sel_ldr_x86_64' ] tools = os.path.join(pepperdir, 'tools') for name in rename_list: src = os.path.join(pepperdir, 'tools', name) dst = os.path.join(pepperdir, 'tools', name + '.exe') buildbot_common.Move(src, dst) if not skip_examples: CopyExamples(pepperdir, toolchains) tarname = 'naclsdk_' + platform + '.bz2' if 'pnacl' in toolchains: tarname = 'p' + tarname tarfile = os.path.join(OUT_DIR, tarname) if not skip_tar: buildbot_common.BuildStep('Tar Pepper Bundle') buildbot_common.Run([ sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile, 'pepper_' + pepper_ver ], cwd=NACL_DIR) # Run build tests buildbot_common.BuildStep('Run build_tools tests') buildbot_common.Run([ sys.executable, os.path.join(SDK_SRC_DIR, 'build_tools', 'tests', 'test_all.py') ]) # build sdk update if not skip_update: build_updater.BuildUpdater(OUT_DIR) # start local server sharing a manifest + the new bundle if not skip_test_updater and not skip_tar: buildbot_common.BuildStep('Move bundle to localserver dir') buildbot_common.MakeDir(SERVER_DIR) buildbot_common.Move(tarfile, SERVER_DIR) tarfile = os.path.join(SERVER_DIR, tarname) server = None try: buildbot_common.BuildStep('Run local server') server = test_server.LocalHTTPServer(SERVER_DIR) buildbot_common.BuildStep('Generate manifest') with open(tarfile, 'rb') as tarfile_stream: archive_sha1, archive_size = manifest_util.DownloadAndComputeHash( tarfile_stream) archive = manifest_util.Archive(manifest_util.GetHostOS()) archive.CopyFrom({ 'url': server.GetURL(tarname), 'size': archive_size, 'checksum': { 'sha1': archive_sha1 } }) bundle = manifest_util.Bundle('pepper_' + pepper_ver) bundle.CopyFrom({ 'revision': int(clnumber), 'repath': 'pepper_' + pepper_ver, 'version': int(pepper_ver), 'description': 'Chrome %s bundle, revision %s' % (pepper_ver, clnumber), 'stability': 'dev', 'recommended': 'no', 'archives': [archive] }) manifest = manifest_util.SDKManifest() manifest.SetBundle(bundle) manifest_name = 'naclsdk_manifest2.json' with open(os.path.join(SERVER_DIR, manifest_name), 'wb') as \ manifest_stream: manifest_stream.write(manifest.GetDataAsString()) # use newly built sdk updater to pull this bundle buildbot_common.BuildStep('Update from local server') naclsdk_sh = os.path.join(OUT_DIR, 'nacl_sdk', 'naclsdk') if platform == 'win': naclsdk_sh += '.bat' buildbot_common.Run([ naclsdk_sh, '-U', server.GetURL(manifest_name), 'update', 'pepper_' + pepper_ver ]) # If we are testing examples, do it in the newly pulled directory. pepperdir = os.path.join(OUT_DIR, 'nacl_sdk', 'pepper_' + pepper_ver) # kill server finally: if server: server.Shutdown() # build examples. if not skip_examples: buildbot_common.BuildStep('Test Build Examples') example_dir = os.path.join(pepperdir, 'examples') makefile = os.path.join(example_dir, 'Makefile') if os.path.isfile(makefile): print "\n\nMake: " + example_dir buildbot_common.Run(['make', '-j8'], cwd=os.path.abspath(example_dir), shell=True) # Archive on non-trybots. buildername = os.environ.get('BUILDBOT_BUILDERNAME', '') if options.archive or '-sdk' in buildername: buildbot_common.BuildStep('Archive build') bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % \ build_utils.ChromeVersion() buildbot_common.Archive(tarname, bucket_path, os.path.dirname(tarfile)) if not skip_update: # Only push up sdk_tools.tgz on the linux buildbot. if buildername == 'linux-sdk-multi': sdk_tools = os.path.join(OUT_DIR, 'sdk_tools.tgz') buildbot_common.Archive('sdk_tools.tgz', bucket_path, OUT_DIR, step_link=False) # generate "manifest snippet" for this archive. if not skip_test_updater: archive = bundle.GetArchive(manifest_util.GetHostOS()) archive.url = 'https://commondatastorage.googleapis.com/' \ 'nativeclient-mirror/nacl/nacl_sdk/%s/%s' % ( build_utils.ChromeVersion(), tarname) manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json') with open(manifest_snippet_file, 'wb') as manifest_snippet_stream: manifest_snippet_stream.write(bundle.GetDataAsString()) buildbot_common.Archive(tarname + '.json', bucket_path, OUT_DIR, step_link=False) return 0
def List(options, argv, config): '''Usage: %prog [global_options] list [options] Lists the available SDK bundles that are available for download.''' def PrintBundle(local_bundle, bundle, needs_update, display_revisions): installed = local_bundle is not None # If bundle is None, there is no longer a remote bundle with this name. if bundle is None: bundle = local_bundle if display_revisions: if needs_update: revision = ' (r%s -> r%s)' % (local_bundle.revision, bundle.revision) else: revision = ' (r%s)' % (bundle.revision,) else: revision = '' InfoPrint(' %s%s %s (%s)%s' % ( 'I' if installed else ' ', '*' if needs_update else ' ', bundle.name, bundle.stability, revision)) DebugPrint("Running List command with: %s, %s" %(options, argv)) parser = optparse.OptionParser(usage=List.__doc__) parser.add_option( '-r', '--revision', dest='revision', default=False, action='store_true', help='display revision numbers') (list_options, _) = parser.parse_args(argv) manifest = LoadManifestFromURLs([options.manifest_url] + config.GetSources()) manifest_path = os.path.join(options.user_data_dir, options.manifest_filename) local_manifest = LoadFromFile(manifest_path, manifest_util.SDKManifest()) any_bundles_need_update = False InfoPrint('Bundles:') InfoPrint(' I: installed\n *: update available\n') for bundle in manifest.GetBundles(): local_bundle = local_manifest.GetBundle(bundle.name) needs_update = local_bundle and local_manifest.BundleNeedsUpdate(bundle) if needs_update: any_bundles_need_update = True PrintBundle(local_bundle, bundle, needs_update, list_options.revision) if not any_bundles_need_update: InfoPrint('\nAll installed bundles are up-to-date.') local_only_bundles = set([b.name for b in local_manifest.GetBundles()]) local_only_bundles -= set([b.name for b in manifest.GetBundles()]) if local_only_bundles: InfoPrint('\nBundles installed locally that are not available remotely:') for bundle_name in local_only_bundles: local_bundle = local_manifest.GetBundle(bundle_name) PrintBundle(local_bundle, None, False, list_options.revision)