def __init__(self, dryrun=False, gsutil=None, verbose=False): super(RealDelegate, self).__init__() self.dryrun = dryrun self.verbose = verbose if gsutil: self.gsutil = gsutil else: self.gsutil = buildbot_common.GetGsutil()
def __init__(self, dryrun=False, gsutil=None, mailfrom=None, mailto=None): super(RealDelegate, self).__init__() self.dryrun = dryrun self.mailfrom = mailfrom self.mailto = mailto if gsutil: self.gsutil = gsutil else: self.gsutil = buildbot_common.GetGsutil()
def update_mono_sdk_json(infos): '''Update the naclmono manifest with the newly built packages''' if len(infos) == 0: return manifest_file = open(MONO_MANIFEST, 'r') mono_manifest = json.loads(manifest_file.read()) manifest_file.close() for info in infos: bundle = {} bundle['name'] = info['naclmono_name'] bundle['description'] = 'Mono for Native Client' bundle['stability'] = info['stability'] bundle['recommended'] = 'no' bundle['version'] = 'experimental' archive = {} sha1_hash = hashlib.sha1() f = open(info['naclmono_name'] + '.bz2', 'rb') sha1_hash.update(f.read()) archive['size'] = f.tell() f.close() archive['checksum'] = {'sha1': sha1_hash.hexdigest()} archive['host_os'] = 'all' archive['url'] = ('https://commondatastorage.googleapis.com/' 'nativeclient-mirror/nacl/nacl_sdk/%s/%s/%s.bz2' % (info['naclmono_name'], info['naclmono_rev'], info['naclmono_name'])) bundle['archives'] = [archive] bundle['revision'] = int(info['naclmono_rev']) bundle['sdk_revision'] = int(info['sdk_revision']) # Insert this new bundle into the manifest, # probably overwriting an existing bundle. for key, value in mono_manifest.items(): if key == 'bundles': existing = filter(lambda b: b['name'] == info['naclmono_name'], value) if len(existing) > 0: loc = value.index(existing[0]) value[loc] = bundle else: value.append(bundle) # Write out the file locally, then upload to its known location. manifest_file = open(MONO_MANIFEST, 'w') manifest_file.write(json.dumps(mono_manifest, sort_keys=False, indent=2)) manifest_file.close() buildbot_common.Run([ buildbot_common.GetGsutil(), 'cp', '-a', 'public-read', MONO_MANIFEST, GS_MANIFEST_PATH + MONO_MANIFEST ])
def _RunGsUtil(self, stdin, *args): """Run gsutil as a subprocess. Args: stdin: If non-None, used as input to the process. *args: Arguments to pass to gsutil. The first argument should be an operation such as ls, cp or cat. Returns: The stdout from the process.""" cmd = [buildbot_common.GetGsutil()] + list(args) if stdin: stdin_pipe = subprocess.PIPE else: stdin_pipe = None process = subprocess.Popen(cmd, stdin=stdin_pipe, stdout=subprocess.PIPE) stdout, _ = process.communicate(stdin) if process.returncode != 0: raise subprocess.CalledProcessError(process.returncode, ' '.join(cmd)) return stdout
def main(args): parser = optparse.OptionParser() parser.add_option('--arch', help='Target architecture', dest='arch', default='x86-32') parser.add_option('--sdk-revision', help='SDK Revision' ' (default=buildbot revision)', dest='sdk_revision', default=None) parser.add_option('--sdk-url', help='SDK Download URL', dest='sdk_url', default=None) parser.add_option('--install-dir', help='Install Directory', dest='install_dir', default='naclmono') (options, args) = parser.parse_args(args[1:]) assert sys.platform.find('linux') != -1 buildbot_revision = os.environ.get('BUILDBOT_REVISION', '') build_prefix = options.arch + ' ' buildbot_common.BuildStep(build_prefix + 'Clean Old SDK') buildbot_common.MakeDir(MONO_BUILD_DIR) buildbot_common.RemoveDir(os.path.join(MONO_BUILD_DIR, 'pepper_*')) buildbot_common.BuildStep(build_prefix + 'Setup New SDK') sdk_dir = None sdk_revision = options.sdk_revision sdk_url = options.sdk_url if not sdk_url: if not sdk_revision: assert buildbot_revision sdk_revision = buildbot_revision.split(':')[0] sdk_url = 'gs://nativeclient-mirror/nacl/nacl_sdk/'\ 'trunk.%s/naclsdk_linux.bz2' % sdk_revision sdk_url = sdk_url.replace('https://commondatastorage.googleapis.com/', 'gs://') sdk_file = sdk_url.split('/')[-1] buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', sdk_url, sdk_file], cwd=MONO_BUILD_DIR) tar_file = None try: tar_file = tarfile.open(os.path.join(MONO_BUILD_DIR, sdk_file)) pepper_dir = os.path.commonprefix(tar_file.getnames()) tar_file.extractall(path=MONO_BUILD_DIR) sdk_dir = os.path.join(MONO_BUILD_DIR, pepper_dir) finally: if tar_file: tar_file.close() assert sdk_dir buildbot_common.BuildStep(build_prefix + 'Checkout Mono') # TODO(elijahtaylor): Get git URL from master/trigger to make this # more flexible for building from upstream and release branches. git_url = 'git://github.com/elijahtaylor/mono.git' git_rev = 'HEAD' if buildbot_revision: git_rev = buildbot_revision.split(':')[1] if not os.path.exists(MONO_DIR): buildbot_common.MakeDir(MONO_DIR) buildbot_common.Run(['git', 'clone', git_url, MONO_DIR]) else: buildbot_common.Run(['git', 'fetch'], cwd=MONO_DIR) if git_rev: buildbot_common.Run(['git', 'checkout', git_rev], cwd=MONO_DIR) arch_to_bitsize = {'x86-32': '32', 'x86-64': '64'} arch_to_output_folder = { 'x86-32': 'runtime-build', 'x86-64': 'runtime64-build' } buildbot_common.BuildStep(build_prefix + 'Configure Mono') os.environ['NACL_SDK_ROOT'] = sdk_dir os.environ['TARGET_BITSIZE'] = arch_to_bitsize[options.arch] buildbot_common.Run(['./autogen.sh'], cwd=MONO_DIR) buildbot_common.Run(['make', 'distclean'], cwd=MONO_DIR) buildbot_common.BuildStep(build_prefix + 'Build and Install Mono') nacl_interp_script = os.path.join(SDK_BUILD_DIR, 'nacl_interp_loader_mono.sh') os.environ['NACL_INTERP_LOADER'] = nacl_interp_script buildbot_common.Run( [ './nacl-mono-runtime.sh', MONO_DIR, # Mono directory with 'configure' arch_to_output_folder[options.arch], # Build dir options.install_dir ], cwd=SDK_BUILD_DIR) buildbot_common.BuildStep(build_prefix + 'Test Mono') buildbot_common.Run(['make', 'check', '-j8'], cwd=os.path.join(SDK_BUILD_DIR, arch_to_output_folder[options.arch])) return 0
def get_sdk_build_info(): '''Returns a list of dictionaries for versions of NaCl Mono to build which are out of date compared to the SDKs available to naclsdk''' # Get a copy of the naclsdk manifest file buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', GS_MANIFEST_PATH + SDK_MANIFEST, '.']) manifest_file = open(SDK_MANIFEST, 'r') sdk_manifest = json.loads(manifest_file.read()) manifest_file.close() pepper_infos = [] for key, value in sdk_manifest.items(): if key == 'bundles': stabilities = ['stable', 'beta', 'dev', 'post_stable'] # Pick pepper_* bundles, need pepper_19 or greater to build Mono bundles = filter(lambda b: (b['stability'] in stabilities and 'pepper_' in b['name']) and b['version'] >= 19, value) for b in bundles: newdict = {} newdict['pepper_revision'] = str(b['version']) linux_arch = filter(lambda u: u['host_os'] == 'linux', b['archives']) newdict['sdk_url'] = linux_arch[0]['url'] newdict['sdk_revision'] = b['revision'] newdict['stability'] = b['stability'] newdict['naclmono_name'] = 'naclmono_' + newdict['pepper_revision'] pepper_infos.append(newdict) # Get a copy of the naclmono manifest file buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', GS_MANIFEST_PATH + MONO_MANIFEST, '.']) manifest_file = open(MONO_MANIFEST, 'r') mono_manifest = json.loads(manifest_file.read()) manifest_file.close() ret = [] mono_manifest_dirty = False # Check to see if we need to rebuild mono based on sdk revision for key, value in mono_manifest.items(): if key == 'bundles': for info in pepper_infos: bundle = filter(lambda b: b['name'] == info['naclmono_name'], value) if len(bundle) == 0: info['naclmono_rev'] = '1' ret.append(info) else: if info['sdk_revision'] != bundle[0]['sdk_revision']: # This bundle exists in the mono manifest, bump the revision # for the new build we're about to make. info['naclmono_rev'] = str(bundle[0]['revision'] + 1) ret.append(info) elif info['stability'] != bundle[0]['stability']: # If all that happened was the SDK bundle was promoted in stability, # change only that and re-write the manifest mono_manifest_dirty = True bundle[0]['stability'] = info['stability'] # re-write the manifest here because there are no bundles to build but # the manifest has changed if mono_manifest_dirty and len(ret) == 0: manifest_file = open(MONO_MANIFEST, 'w') manifest_file.write(json.dumps(mono_manifest, sort_keys=False, indent=2)) manifest_file.close() buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', '-a', 'public-read', MONO_MANIFEST, GS_MANIFEST_PATH + MONO_MANIFEST]) return ret
def main(args): parser = optparse.OptionParser() parser.add_option('--arch', help='Target architecture', dest='arch', default='x86-32') parser.add_option('--sdk-revision', help='SDK Revision' ' (default=buildbot revision)', dest='sdk_revision', default=None) parser.add_option('--sdk-url', help='SDK Download URL', dest='sdk_url', default=None) parser.add_option('--install-dir', help='Install Directory', dest='install_dir', default='naclmono') (options, args) = parser.parse_args(args[1:]) assert sys.platform.find('linux') != -1 buildbot_revision = os.environ.get('BUILDBOT_REVISION', '') build_prefix = options.arch + ' ' buildbot_common.BuildStep(build_prefix + 'Clean Old SDK') buildbot_common.MakeDir(MONO_BUILD_DIR) buildbot_common.RemoveDir(os.path.join(MONO_BUILD_DIR, 'pepper_*')) buildbot_common.BuildStep(build_prefix + 'Setup New SDK') sdk_dir = None sdk_revision = options.sdk_revision sdk_url = options.sdk_url if not sdk_url: if not sdk_revision: assert buildbot_revision sdk_revision = buildbot_revision.split(':')[0] sdk_url = 'gs://nativeclient-mirror/nacl/nacl_sdk/'\ 'trunk.%s/naclsdk_linux.tar.bz2' % sdk_revision sdk_url = sdk_url.replace('https://commondatastorage.googleapis.com/', 'gs://') sdk_file = sdk_url.split('/')[-1] buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', sdk_url, sdk_file], cwd=MONO_BUILD_DIR) tar_file = None try: tar_file = tarfile.open(os.path.join(MONO_BUILD_DIR, sdk_file)) pepper_dir = os.path.commonprefix(tar_file.getnames()) tar_file.extractall(path=MONO_BUILD_DIR) sdk_dir = os.path.join(MONO_BUILD_DIR, pepper_dir) finally: if tar_file: tar_file.close() assert sdk_dir buildbot_common.BuildStep(build_prefix + 'Checkout Mono') # TODO(elijahtaylor): Get git URL from master/trigger to make this # more flexible for building from upstream and release branches. if options.arch == 'arm': git_url = 'git://github.com/igotti-google/mono.git' git_rev = 'arm_nacl' else: git_url = 'git://github.com/elijahtaylor/mono.git' git_rev = 'HEAD' if buildbot_revision: # Unfortunately, we use different git branches/revisions # for ARM and x86 now, so ignore buildbot_revision variable for ARM. # Need to rethink this approach, if we'll plan to support # more flexible repo selection mechanism. if options.arch != 'arm': git_rev = buildbot_revision.split(':')[1] # ARM and x86 is built out of different git trees, so distinguish # them by appending the arch. It also makes 32 and 64 bit x86 separated, # which is good. # TODO(olonho): maybe we need to avoid modifications of global. global MONO_DIR tag = options.arch MONO_DIR = "%s-%s" % (MONO_DIR, tag) if not os.path.exists(MONO_DIR): buildbot_common.MakeDir(MONO_DIR) buildbot_common.Run(['git', 'clone', git_url, MONO_DIR]) else: buildbot_common.Run(['git', 'fetch'], cwd=MONO_DIR) if git_rev: buildbot_common.Run(['git', 'checkout', git_rev], cwd=MONO_DIR) arch_to_bitsize = {'x86-32': '32', 'x86-64': '64', 'arm': 'arm'} arch_to_output_folder = { 'x86-32': 'runtime-x86-32-build', 'x86-64': 'runtime-x86-64-build', 'arm': 'runtime-arm-build' } buildbot_common.BuildStep(build_prefix + 'Configure Mono') os.environ['NACL_SDK_ROOT'] = sdk_dir os.environ['TARGET_ARCH'] = options.arch os.environ['TARGET_BITSIZE'] = arch_to_bitsize[options.arch] buildbot_common.Run(['./autogen.sh'], cwd=MONO_DIR) buildbot_common.Run(['make', 'distclean'], cwd=MONO_DIR) buildbot_common.BuildStep(build_prefix + 'Build and Install Mono') nacl_interp_script = os.path.join(SDK_BUILD_DIR, 'nacl_interp_loader_mono.sh') os.environ['NACL_INTERP_LOADER'] = nacl_interp_script buildbot_common.Run( [ './nacl-mono-runtime.sh', MONO_DIR, # Mono directory with 'configure' arch_to_output_folder[options.arch], # Build dir options.install_dir ], cwd=SDK_BUILD_DIR) # TODO(elijahtaylor,olonho): Re-enable tests on arm when they compile/run. if options.arch != 'arm': buildbot_common.BuildStep(build_prefix + 'Test Mono') buildbot_common.Run(['make', 'check', '-j8'], cwd=os.path.join( SDK_BUILD_DIR, arch_to_output_folder[options.arch])) return 0
def __init__(self, dryrun=False, gsutil=None): self.dryrun = dryrun if gsutil: self.gsutil = gsutil else: self.gsutil = buildbot_common.GetGsutil()