def SyncTgz(url, target, username=None, password=None, verbose=True): """Download a file from a remote server. Args: url: A URL to download from. target: Directory to extract to and prefix to use for tgz file. username: Optional username for download. password: Optional password for download (ignored if no username). verbose: Flag indicating if status shut be printed. """ shutil.rmtree(target, True) tgz_filename = target + '/.tgz' if verbose: print 'Downloading %s to %s...' % (url, tgz_filename) http_download.HttpDownload(url, tgz_filename, username=username, password=password) if verbose: print 'Extracting from %s...' % tgz_filename if sys.platform == 'win32': os.makedirs(os.path.join(target, 'tmptar')) tarfiles = [ 'cyggcc_s-1.dll', 'cygiconv-2.dll', 'cygintl-8.dll', 'cygwin1.dll', 'gzip.exe', 'tar.exe' ] for filename in tarfiles: http_download.HttpDownload( 'http://build.chromium.org/mirror/nacl/cygwin_mirror/cygwin/' + filename, os.path.join(target, 'tmptar', filename)) saveddir = os.getcwd() os.chdir(target) if verbose: verbosechar = 'v' else: verbosechar = '' os.spawnv(os.P_WAIT, os.path.join(target, 'tmptar', 'tar.exe'), [ '/tmptar/tar', '--use-compress-program', '/tmptar/gzip', '-xS' + verbosechar + 'pf', '../.tgz' ]) os.chdir(saveddir) # Some antivirus software can prevent the removal - print message, but # don't stop. for filename in tarfiles: try: os.remove(os.path.join(target, 'tmptar', filename)) except EnvironmentError, e: if verbose: print "Can not remove %s: %s" % (filename, e.strerror) try: os.rmdir(os.path.join(target, 'tmptar')) except EnvironmentError, e: if verbose: print "Can not rmdir %s: %s" % (os.path.join( target, 'tmptar'), e.strerror)
def UpdateHelpers(pepperdir, clobber=False): tools_dir = os.path.join(pepperdir, 'tools') if not os.path.exists(tools_dir): buildbot_common.ErrorExit('SDK tools dir is missing: %s' % tools_dir) exampledir = os.path.join(pepperdir, 'examples') if clobber: buildbot_common.RemoveDir(exampledir) buildbot_common.MakeDir(exampledir) # Copy files for individual build and landing page files = ['favicon.ico', 'httpd.cmd', 'index.css', 'index.js', 'button_close.png', 'button_close_hover.png'] CopyFilesFromTo(files, SDK_RESOURCE_DIR, exampledir) # Copy tools scripts and make includes buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'), tools_dir) buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.mk'), tools_dir) # On Windows add a prebuilt make if getos.GetPlatform() == 'win': buildbot_common.BuildStep('Add MAKE') http_download.HttpDownload(GSTORE + MAKE, os.path.join(tools_dir, 'make.exe'))
def UpdateHelpers(pepperdir, platform, clobber=False): if not os.path.exists(os.path.join(pepperdir, 'tools')): buildbot_common.ErrorExit('Examples depend on missing tools.') exampledir = os.path.join(pepperdir, 'examples') if clobber: buildbot_common.RemoveDir(exampledir) buildbot_common.MakeDir(exampledir) # Copy files for individual bild and landing page files = ['favicon.ico', 'httpd.cmd'] CopyFilesFromTo(files, SDK_EXAMPLE_DIR, exampledir) resourcesdir = os.path.join(SDK_EXAMPLE_DIR, 'resources') files = [ 'index.css', 'index.js', 'button_close.png', 'button_close_hover.png' ] CopyFilesFromTo(files, resourcesdir, exampledir) # Copy tools scripts and make includes buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'), os.path.join(pepperdir, 'tools')) buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.mk'), os.path.join(pepperdir, 'tools')) # On Windows add a prebuilt make if platform == 'win': buildbot_common.BuildStep('Add MAKE') http_download.HttpDownload( GSTORE + MAKE, os.path.join(pepperdir, 'tools', 'make.exe'))
def testFetch(self): ioloop = MockIoloop() username = '******' password = '******' url = 'scheme://host:port/' dl = http_download.HttpDownload( url, username=username, password=password, download_complete_cb=self.downloadCallback, ioloop=ioloop) dl.fetch() self.assertEqual(len(mock_http_clients), 1) ht = mock_http_clients[0] self.assertTrue(ht.did_fetch) self.assertTrue(ht.request is not None) self.assertEqual(ht.request.auth_username, username) self.assertEqual(ht.request.auth_password, password) self.assertEqual(ht.request.url, url) resp = MockHttpResponse(418) ht.callback(resp) self.assertEqual(self.dl_cb_faultcode, 9010) # DOWNLOAD_FAILED self.assertTrue(self.dl_cb_faultstring) self.assertFalse(self.dl_cb_filename)
def SyncURL(url, filename=None, stamp_dir=None, min_time=None, hash_val=None, keep=False, verbose=False, stamp_index=0): """Synchronize a destination file with a URL if the URL does not match the URL stamp, then we must re-download it. Arugments: url: the url which will to compare against and download filename: the file to create on download path: the download path stamp_dir: the filename containing the URL stamp to check against hash_val: if set, the expected hash which must be matched verbose: prints out status as it runs stamp_index: index within the stamp file to check. Returns: True if the file is replaced False if the file is not replaced Exception: HashError: if the hash does not match """ assert url and filename # If we are not keeping the tarball, or we already have it, we can # skip downloading it for this reason. If we are keeping it, # it must exist. if keep: tarball_ok = os.path.isfile(filename) else: tarball_ok = True # If we don't need the tarball and the stamp_file matches the url, then # we must be up to date. If the URL differs but the recorded hash matches # the one we'll insist the tarball has, then that's good enough too. # TODO(mcgrathr): Download the .sha1sum file first to compare with # the cached hash, in case --file-hash options weren't used. if tarball_ok and stamp_dir is not None: if StampIsCurrent(stamp_dir, SOURCE_STAMP, url, min_time): if verbose: print '%s is already up to date.' % filename return False if (hash_val is not None and StampIsCurrent(stamp_dir, HASH_STAMP, hash_val, min_time, stamp_index)): if verbose: print '%s is identical to the up to date file.' % filename return False if verbose: print 'Updating %s\n\tfrom %s.' % (filename, url) EnsureFileCanBeWritten(filename) http_download.HttpDownload(url, filename) if hash_val: tar_hash = HashFile(filename) if hash_val != tar_hash: raise HashError(actual_hash=tar_hash, expected_hash=hash_val, download_url=url) return True
def BuildStepCopyBuildHelpers(pepperdir, platform): buildbot_common.BuildStep('Copy build helpers') buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'), os.path.join(pepperdir, 'tools')) if platform == 'win': buildbot_common.BuildStep('Add MAKE') http_download.HttpDownload(GSTORE + MAKE, os.path.join(pepperdir, 'tools', 'make.exe'))
def HttpDownload(url, target): """Default download route.""" http_download.HttpDownload(url, os.path.abspath(target), verbose=False, logger=logging.debug)
def main(args): parser = optparse.OptionParser() parser.add_option('--pnacl', help='Enable pnacl build.', action='store_true', dest='pnacl', default=False) parser.add_option('--examples', help='Rebuild the examples.', action='store_true', dest='examples', default=False) parser.add_option('--update', help='Rebuild the updater.', action='store_true', dest='update', default=False) parser.add_option('--skip-tar', help='Skip generating a tarball.', action='store_true', dest='skip_tar', default=False) parser.add_option('--archive', help='Force the archive step.', action='store_true', dest='archive', default=False) parser.add_option('--release', help='PPAPI release version.', dest='release', default=None) options, args = parser.parse_args(args[1:]) platform = getos.GetPlatform() arch = 'x86' builder_name = os.getenv('BUILDBOT_BUILDERNAME', '') if builder_name.find('pnacl') >= 0 and builder_name.find('sdk') >= 0: options.pnacl = True if options.pnacl: toolchains = ['pnacl'] else: toolchains = ['newlib', 'glibc'] print 'Building: ' + ' '.join(toolchains) skip = options.examples or options.update skip_examples = skip skip_update = skip skip_untar = skip skip_build = skip skip_tar = skip or options.skip_tar if options.examples: skip_examples = False skip_update = not options.update if options.archive and (options.examples or options.skip_tar): parser.error('Incompatible arguments with archive.') pepper_ver = str(int(build_utils.ChromeMajorVersion())) clnumber = lastchange.FetchVersionInfo(None).revision if options.release: pepper_ver = options.release print 'Building PEPPER %s at %s' % (pepper_ver, clnumber) if not skip_build: buildbot_common.BuildStep('Rerun hooks to get toolchains') buildbot_common.Run(['gclient', 'runhooks'], cwd=SRC_DIR, shell=(platform == 'win')) buildbot_common.BuildStep('Clean Pepper Dir') pepperdir = os.path.join(SRC_DIR, 'out', 'pepper_' + pepper_ver) if not skip_untar: buildbot_common.RemoveDir(pepperdir) buildbot_common.MakeDir(os.path.join(pepperdir, 'toolchain')) buildbot_common.MakeDir(os.path.join(pepperdir, 'tools')) buildbot_common.BuildStep('Add Text Files') files = ['AUTHORS', 'COPYING', 'LICENSE', 'NOTICE', 'README'] files = [os.path.join(SDK_SRC_DIR, filename) for filename in files] oshelpers.Copy(['-v'] + files + [pepperdir]) # Clean out the temporary toolchain untar directory if not skip_untar: UntarToolchains(pepperdir, platform, arch, toolchains) if not skip_build: BuildToolchains(pepperdir, platform, arch, pepper_ver, toolchains) buildbot_common.BuildStep('Copy make OS helpers') buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'), os.path.join(pepperdir, 'tools')) if platform == 'win': buildbot_common.BuildStep('Add MAKE') http_download.HttpDownload( GSTORE + MAKE, os.path.join(pepperdir, 'tools', 'make.exe')) if not skip_examples: CopyExamples(pepperdir, toolchains) if not skip_tar: buildbot_common.BuildStep('Tar Pepper Bundle') tarname = 'naclsdk_' + platform + '.bz2' if 'pnacl' in toolchains: tarname = 'p' + tarname tarfile = os.path.join(OUT_DIR, tarname) buildbot_common.Run([ sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile, 'pepper_' + pepper_ver ], cwd=NACL_DIR) # Archive on non-trybots. if options.archive or '-sdk' in os.environ.get('BUILDBOT_BUILDERNAME', ''): buildbot_common.BuildStep('Archive build') buildbot_common.Archive( tarname, 'nativeclient-mirror/nacl/nacl_sdk/%s' % build_utils.ChromeVersion(), OUT_DIR) if not skip_examples: buildbot_common.BuildStep('Test Build Examples') filelist = os.listdir(os.path.join(pepperdir, 'examples')) for filenode in filelist: dirnode = os.path.join(pepperdir, 'examples', filenode) makefile = os.path.join(dirnode, 'Makefile') if os.path.isfile(makefile): print "\n\nMake: " + dirnode buildbot_common.Run(['make', 'all', '-j8'], cwd=os.path.abspath(dirnode), shell=True) # Build SDK Tools if not skip_update: BuildUpdater() return 0
def HttpDownload(url, target): """Default download route.""" http_download.HttpDownload(url, os.path.abspath(target), logger=logging.info)
def main(args): parser = optparse.OptionParser() parser.add_option('--pnacl', help='Enable pnacl build.', action='store_true', dest='pnacl', default=False) parser.add_option('--examples', help='Only build the examples.', action='store_true', dest='only_examples', default=False) parser.add_option('--update', help='Only build the updater.', action='store_true', dest='only_updater', default=False) parser.add_option('--skip-tar', help='Skip generating a tarball.', action='store_true', dest='skip_tar', default=False) parser.add_option('--archive', help='Force the archive step.', action='store_true', dest='archive', default=False) parser.add_option('--release', help='PPAPI release version.', dest='release', default=None) options, args = parser.parse_args(args[1:]) platform = getos.GetPlatform() arch = 'x86' builder_name = os.getenv('BUILDBOT_BUILDERNAME', '') if builder_name.find('pnacl') >= 0 and builder_name.find('sdk') >= 0: options.pnacl = True if options.pnacl: toolchains = ['pnacl'] else: toolchains = ['newlib', 'glibc'] print 'Building: ' + ' '.join(toolchains) skip = options.only_examples or options.only_updater skip_examples = skip and not options.only_examples skip_update = skip and not options.only_updater skip_untar = skip skip_build = skip skip_test_updater = skip skip_tar = skip or options.skip_tar if options.archive and (options.only_examples or options.skip_tar): parser.error('Incompatible arguments with archive.') pepper_ver = str(int(build_utils.ChromeMajorVersion())) pepper_old = str(int(build_utils.ChromeMajorVersion()) - 1) clnumber = build_utils.ChromeRevision() if options.release: pepper_ver = options.release print 'Building PEPPER %s at %s' % (pepper_ver, clnumber) if not skip_build: buildbot_common.BuildStep('Rerun hooks to get toolchains') buildbot_common.Run(['gclient', 'runhooks'], cwd=SRC_DIR, shell=(platform == 'win')) buildbot_common.BuildStep('Clean Pepper Dirs') pepperdir = os.path.join(SRC_DIR, 'out', 'pepper_' + pepper_ver) pepperold = os.path.join(SRC_DIR, 'out', 'pepper_' + pepper_old) buildbot_common.RemoveDir(pepperold) if not skip_untar: buildbot_common.RemoveDir(pepperdir) buildbot_common.MakeDir(os.path.join(pepperdir, 'libraries')) buildbot_common.MakeDir(os.path.join(pepperdir, 'toolchain')) buildbot_common.MakeDir(os.path.join(pepperdir, 'tools')) else: buildbot_common.MakeDir(pepperdir) if not skip_build: buildbot_common.BuildStep('Add Text Files') files = ['AUTHORS', 'COPYING', 'LICENSE', 'NOTICE', 'README'] files = [os.path.join(SDK_SRC_DIR, filename) for filename in files] oshelpers.Copy(['-v'] + files + [pepperdir]) # Clean out the temporary toolchain untar directory if not skip_untar: UntarToolchains(pepperdir, platform, arch, toolchains) if not skip_build: BuildToolchains(pepperdir, platform, arch, pepper_ver, toolchains) InstallHeaders(os.path.join(pepperdir, 'libraries'), pepper_ver, 'libs') if not skip_build: buildbot_common.BuildStep('Copy make OS helpers') buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'), os.path.join(pepperdir, 'tools')) if platform == 'win': buildbot_common.BuildStep('Add MAKE') http_download.HttpDownload( GSTORE + MAKE, os.path.join(pepperdir, 'tools', 'make.exe')) rename_list = [ 'ncval_x86_32', 'ncval_x86_64', 'sel_ldr_x86_32', 'sel_ldr_x86_64' ] tools = os.path.join(pepperdir, 'tools') for name in rename_list: src = os.path.join(pepperdir, 'tools', name) dst = os.path.join(pepperdir, 'tools', name + '.exe') buildbot_common.Move(src, dst) if not skip_examples: CopyExamples(pepperdir, toolchains) tarname = 'naclsdk_' + platform + '.bz2' if 'pnacl' in toolchains: tarname = 'p' + tarname tarfile = os.path.join(OUT_DIR, tarname) if not skip_tar: buildbot_common.BuildStep('Tar Pepper Bundle') buildbot_common.Run([ sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile, 'pepper_' + pepper_ver ], cwd=NACL_DIR) # Run build tests buildbot_common.BuildStep('Run build_tools tests') buildbot_common.Run([ sys.executable, os.path.join(SDK_SRC_DIR, 'build_tools', 'tests', 'test_all.py') ]) # build sdk update if not skip_update: build_updater.BuildUpdater(OUT_DIR) # start local server sharing a manifest + the new bundle if not skip_test_updater and not skip_tar: buildbot_common.BuildStep('Move bundle to localserver dir') buildbot_common.MakeDir(SERVER_DIR) buildbot_common.Move(tarfile, SERVER_DIR) tarfile = os.path.join(SERVER_DIR, tarname) server = None try: buildbot_common.BuildStep('Run local server') server = test_server.LocalHTTPServer(SERVER_DIR) buildbot_common.BuildStep('Generate manifest') with open(tarfile, 'rb') as tarfile_stream: archive_sha1, archive_size = manifest_util.DownloadAndComputeHash( tarfile_stream) archive = manifest_util.Archive(manifest_util.GetHostOS()) archive.CopyFrom({ 'url': server.GetURL(tarname), 'size': archive_size, 'checksum': { 'sha1': archive_sha1 } }) bundle = manifest_util.Bundle('pepper_' + pepper_ver) bundle.CopyFrom({ 'revision': int(clnumber), 'repath': 'pepper_' + pepper_ver, 'version': int(pepper_ver), 'description': 'Chrome %s bundle, revision %s' % (pepper_ver, clnumber), 'stability': 'dev', 'recommended': 'no', 'archives': [archive] }) manifest = manifest_util.SDKManifest() manifest.SetBundle(bundle) manifest_name = 'naclsdk_manifest2.json' with open(os.path.join(SERVER_DIR, manifest_name), 'wb') as \ manifest_stream: manifest_stream.write(manifest.GetDataAsString()) # use newly built sdk updater to pull this bundle buildbot_common.BuildStep('Update from local server') naclsdk_sh = os.path.join(OUT_DIR, 'nacl_sdk', 'naclsdk') if platform == 'win': naclsdk_sh += '.bat' buildbot_common.Run([ naclsdk_sh, '-U', server.GetURL(manifest_name), 'update', 'pepper_' + pepper_ver ]) # If we are testing examples, do it in the newly pulled directory. pepperdir = os.path.join(OUT_DIR, 'nacl_sdk', 'pepper_' + pepper_ver) # kill server finally: if server: server.Shutdown() # build examples. if not skip_examples: buildbot_common.BuildStep('Test Build Examples') example_dir = os.path.join(pepperdir, 'examples') makefile = os.path.join(example_dir, 'Makefile') if os.path.isfile(makefile): print "\n\nMake: " + example_dir buildbot_common.Run(['make', '-j8'], cwd=os.path.abspath(example_dir), shell=True) # Archive on non-trybots. buildername = os.environ.get('BUILDBOT_BUILDERNAME', '') if options.archive or '-sdk' in buildername: buildbot_common.BuildStep('Archive build') bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % \ build_utils.ChromeVersion() buildbot_common.Archive(tarname, bucket_path, os.path.dirname(tarfile)) if not skip_update: # Only push up sdk_tools.tgz on the linux buildbot. if buildername == 'linux-sdk-multi': sdk_tools = os.path.join(OUT_DIR, 'sdk_tools.tgz') buildbot_common.Archive('sdk_tools.tgz', bucket_path, OUT_DIR, step_link=False) # generate "manifest snippet" for this archive. if not skip_test_updater: archive = bundle.GetArchive(manifest_util.GetHostOS()) archive.url = 'https://commondatastorage.googleapis.com/' \ 'nativeclient-mirror/nacl/nacl_sdk/%s/%s' % ( build_utils.ChromeVersion(), tarname) manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json') with open(manifest_snippet_file, 'wb') as manifest_snippet_stream: manifest_snippet_stream.write(bundle.GetDataAsString()) buildbot_common.Archive(tarname + '.json', bucket_path, OUT_DIR, step_link=False) return 0