def _MakeDummyArchive(self, bundle_name, tarname=None, filename='dummy.txt'): tarname = (tarname or bundle_name) + '.tar.bz2' temp_dir = tempfile.mkdtemp(prefix='archive') try: dummy_path = os.path.join(temp_dir, filename) with open(dummy_path, 'w') as stream: stream.write('Dummy stuff for %s' % bundle_name) # Build the tarfile directly into the server's directory. tar_path = os.path.join(self.basedir, tarname) tarstream = tarfile.open(tar_path, 'w:bz2') try: tarstream.add(dummy_path, os.path.join(bundle_name, filename)) finally: tarstream.close() with open(tar_path, 'rb') as archive_stream: sha1, size = manifest_util.DownloadAndComputeHash( archive_stream) archive = manifest_util.Archive(manifest_util.GetHostOS()) archive.url = self.server.GetURL(os.path.basename(tar_path)) archive.size = size archive.checksum = sha1 return archive finally: oshelpers.Remove(['-rf', temp_dir])
def GetManifestBundle(pepper_ver, revision, tarfile, archive_url): with open(tarfile, 'rb') as tarfile_stream: archive_sha1, archive_size = manifest_util.DownloadAndComputeHash( tarfile_stream) archive = manifest_util.Archive(manifest_util.GetHostOS()) archive.url = archive_url archive.size = archive_size archive.checksum = archive_sha1 bundle = manifest_util.Bundle('pepper_' + pepper_ver) bundle.revision = int(revision) bundle.repath = 'pepper_' + pepper_ver bundle.version = int(pepper_ver) bundle.description = 'Chrome %s bundle, revision %s' % (pepper_ver, revision) bundle.stability = 'dev' bundle.recommended = 'no' bundle.archives = [archive] return bundle
def _BuildUpdaterArchive(self, rel_path, revision): """Build a new sdk_tools bundle. Args: rel_path: The relative path to build the updater. revision: The revision number to give to this bundle. Returns: A manifest_util.Archive() that points to this new bundle on the local server. """ self._BuildUpdater(os.path.join(self.basedir, rel_path), revision) new_sdk_tools_tgz = os.path.join(self.basedir, rel_path, 'sdk_tools.tgz') with open(new_sdk_tools_tgz, 'rb') as sdk_tools_stream: archive_sha1, archive_size = manifest_util.DownloadAndComputeHash( sdk_tools_stream) archive = manifest_util.Archive('all') archive.url = self.server.GetURL('%s/sdk_tools.tgz' % (rel_path,)) archive.checksum = archive_sha1 archive.size = archive_size return archive
def LoadManifestFromURLs(urls): '''Returns a manifest loaded from |urls|, merged into one manifest.''' manifest = manifest_util.SDKManifest() for url in urls: try: url_stream = UrlOpen(url) except urllib2.URLError as e: raise Error('Unable to open %s. [%s]' % (url, e)) manifest_stream = cStringIO.StringIO() manifest_util.DownloadAndComputeHash(url_stream, manifest_stream) temp_manifest = manifest_util.SDKManifest() temp_manifest.LoadDataFromString(manifest_stream.getvalue()) manifest.MergeManifest(temp_manifest) def BundleFilter(bundle): # Only add this bundle if it's supported on this platform. return bundle.GetHostOSArchive() manifest.FilterBundles(BundleFilter) return manifest
def DownloadArchiveToFile(archive, dest_path): '''Download the archive's data to a file at dest_path. As a side effect, computes the sha1 hash and data size, both returned as a tuple. Raises an Error if the url can't be opened, or an IOError exception if dest_path can't be opened. Args: dest_path: Path for the file that will receive the data. Return: A tuple (sha1, size) with the sha1 hash and data size respectively.''' sha1 = None size = 0 with open(dest_path, 'wb') as to_stream: from_stream = None try: from_stream = UrlOpen(archive.url) except urllib2.URLError: raise Error('Cannot open "%s" for archive %s' % (archive.url, archive.host_os)) try: content_length = int(from_stream.info()[HTTP_CONTENT_LENGTH]) progress_function = ProgressFunction( content_length).GetProgressFunction() InfoPrint('Downloading %s' % archive.url) sha1, size = manifest_util.DownloadAndComputeHash( from_stream, to_stream=to_stream, progress_func=progress_function) if size != content_length: raise Error('Download size mismatch for %s.\n' 'Expected %s bytes but got %s' % (archive.url, content_length, size)) finally: if from_stream: from_stream.close() return sha1, size
def main(args): parser = optparse.OptionParser() parser.add_option('--pnacl', help='Enable pnacl build.', action='store_true', dest='pnacl', default=False) parser.add_option('--examples', help='Only build the examples.', action='store_true', dest='only_examples', default=False) parser.add_option('--update', help='Only build the updater.', action='store_true', dest='only_updater', default=False) parser.add_option('--skip-tar', help='Skip generating a tarball.', action='store_true', dest='skip_tar', default=False) parser.add_option('--archive', help='Force the archive step.', action='store_true', dest='archive', default=False) parser.add_option('--release', help='PPAPI release version.', dest='release', default=None) options, args = parser.parse_args(args[1:]) platform = getos.GetPlatform() arch = 'x86' builder_name = os.getenv('BUILDBOT_BUILDERNAME', '') if builder_name.find('pnacl') >= 0 and builder_name.find('sdk') >= 0: options.pnacl = True if options.pnacl: toolchains = ['pnacl'] else: toolchains = ['newlib', 'glibc'] print 'Building: ' + ' '.join(toolchains) skip = options.only_examples or options.only_updater skip_examples = skip and not options.only_examples skip_update = skip and not options.only_updater skip_untar = skip skip_build = skip skip_test_updater = skip skip_tar = skip or options.skip_tar if options.archive and (options.only_examples or options.skip_tar): parser.error('Incompatible arguments with archive.') pepper_ver = str(int(build_utils.ChromeMajorVersion())) pepper_old = str(int(build_utils.ChromeMajorVersion()) - 1) clnumber = build_utils.ChromeRevision() if options.release: pepper_ver = options.release print 'Building PEPPER %s at %s' % (pepper_ver, clnumber) if not skip_build: buildbot_common.BuildStep('Rerun hooks to get toolchains') buildbot_common.Run(['gclient', 'runhooks'], cwd=SRC_DIR, shell=(platform == 'win')) buildbot_common.BuildStep('Clean Pepper Dirs') pepperdir = os.path.join(SRC_DIR, 'out', 'pepper_' + pepper_ver) pepperold = os.path.join(SRC_DIR, 'out', 'pepper_' + pepper_old) buildbot_common.RemoveDir(pepperold) if not skip_untar: buildbot_common.RemoveDir(pepperdir) buildbot_common.MakeDir(os.path.join(pepperdir, 'libraries')) buildbot_common.MakeDir(os.path.join(pepperdir, 'toolchain')) buildbot_common.MakeDir(os.path.join(pepperdir, 'tools')) else: buildbot_common.MakeDir(pepperdir) if not skip_build: buildbot_common.BuildStep('Add Text Files') files = ['AUTHORS', 'COPYING', 'LICENSE', 'NOTICE', 'README'] files = [os.path.join(SDK_SRC_DIR, filename) for filename in files] oshelpers.Copy(['-v'] + files + [pepperdir]) # Clean out the temporary toolchain untar directory if not skip_untar: UntarToolchains(pepperdir, platform, arch, toolchains) if not skip_build: BuildToolchains(pepperdir, platform, arch, pepper_ver, toolchains) InstallHeaders(os.path.join(pepperdir, 'libraries'), pepper_ver, 'libs') if not skip_build: buildbot_common.BuildStep('Copy make OS helpers') buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'), os.path.join(pepperdir, 'tools')) if platform == 'win': buildbot_common.BuildStep('Add MAKE') http_download.HttpDownload( GSTORE + MAKE, os.path.join(pepperdir, 'tools', 'make.exe')) rename_list = [ 'ncval_x86_32', 'ncval_x86_64', 'sel_ldr_x86_32', 'sel_ldr_x86_64' ] tools = os.path.join(pepperdir, 'tools') for name in rename_list: src = os.path.join(pepperdir, 'tools', name) dst = os.path.join(pepperdir, 'tools', name + '.exe') buildbot_common.Move(src, dst) if not skip_examples: CopyExamples(pepperdir, toolchains) tarname = 'naclsdk_' + platform + '.bz2' if 'pnacl' in toolchains: tarname = 'p' + tarname tarfile = os.path.join(OUT_DIR, tarname) if not skip_tar: buildbot_common.BuildStep('Tar Pepper Bundle') buildbot_common.Run([ sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile, 'pepper_' + pepper_ver ], cwd=NACL_DIR) # Run build tests buildbot_common.BuildStep('Run build_tools tests') buildbot_common.Run([ sys.executable, os.path.join(SDK_SRC_DIR, 'build_tools', 'tests', 'test_all.py') ]) # build sdk update if not skip_update: build_updater.BuildUpdater(OUT_DIR) # start local server sharing a manifest + the new bundle if not skip_test_updater and not skip_tar: buildbot_common.BuildStep('Move bundle to localserver dir') buildbot_common.MakeDir(SERVER_DIR) buildbot_common.Move(tarfile, SERVER_DIR) tarfile = os.path.join(SERVER_DIR, tarname) server = None try: buildbot_common.BuildStep('Run local server') server = test_server.LocalHTTPServer(SERVER_DIR) buildbot_common.BuildStep('Generate manifest') with open(tarfile, 'rb') as tarfile_stream: archive_sha1, archive_size = manifest_util.DownloadAndComputeHash( tarfile_stream) archive = manifest_util.Archive(manifest_util.GetHostOS()) archive.CopyFrom({ 'url': server.GetURL(tarname), 'size': archive_size, 'checksum': { 'sha1': archive_sha1 } }) bundle = manifest_util.Bundle('pepper_' + pepper_ver) bundle.CopyFrom({ 'revision': int(clnumber), 'repath': 'pepper_' + pepper_ver, 'version': int(pepper_ver), 'description': 'Chrome %s bundle, revision %s' % (pepper_ver, clnumber), 'stability': 'dev', 'recommended': 'no', 'archives': [archive] }) manifest = manifest_util.SDKManifest() manifest.SetBundle(bundle) manifest_name = 'naclsdk_manifest2.json' with open(os.path.join(SERVER_DIR, manifest_name), 'wb') as \ manifest_stream: manifest_stream.write(manifest.GetDataAsString()) # use newly built sdk updater to pull this bundle buildbot_common.BuildStep('Update from local server') naclsdk_sh = os.path.join(OUT_DIR, 'nacl_sdk', 'naclsdk') if platform == 'win': naclsdk_sh += '.bat' buildbot_common.Run([ naclsdk_sh, '-U', server.GetURL(manifest_name), 'update', 'pepper_' + pepper_ver ]) # If we are testing examples, do it in the newly pulled directory. pepperdir = os.path.join(OUT_DIR, 'nacl_sdk', 'pepper_' + pepper_ver) # kill server finally: if server: server.Shutdown() # build examples. if not skip_examples: buildbot_common.BuildStep('Test Build Examples') example_dir = os.path.join(pepperdir, 'examples') makefile = os.path.join(example_dir, 'Makefile') if os.path.isfile(makefile): print "\n\nMake: " + example_dir buildbot_common.Run(['make', '-j8'], cwd=os.path.abspath(example_dir), shell=True) # Archive on non-trybots. buildername = os.environ.get('BUILDBOT_BUILDERNAME', '') if options.archive or '-sdk' in buildername: buildbot_common.BuildStep('Archive build') bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % \ build_utils.ChromeVersion() buildbot_common.Archive(tarname, bucket_path, os.path.dirname(tarfile)) if not skip_update: # Only push up sdk_tools.tgz on the linux buildbot. if buildername == 'linux-sdk-multi': sdk_tools = os.path.join(OUT_DIR, 'sdk_tools.tgz') buildbot_common.Archive('sdk_tools.tgz', bucket_path, OUT_DIR, step_link=False) # generate "manifest snippet" for this archive. if not skip_test_updater: archive = bundle.GetArchive(manifest_util.GetHostOS()) archive.url = 'https://commondatastorage.googleapis.com/' \ 'nativeclient-mirror/nacl/nacl_sdk/%s/%s' % ( build_utils.ChromeVersion(), tarname) manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json') with open(manifest_snippet_file, 'wb') as manifest_snippet_stream: manifest_snippet_stream.write(bundle.GetDataAsString()) buildbot_common.Archive(tarname + '.json', bucket_path, OUT_DIR, step_link=False) return 0