def __init__(self, dryrun=False, gsutil=None, verbose=False):
super(RealDelegate, self).__init__()
self.dryrun = dryrun
self.verbose = verbose
if gsutil:
self.gsutil = gsutil
else:
self.gsutil = buildbot_common.GetGsutil()
def __init__(self, dryrun=False, gsutil=None, mailfrom=None, mailto=None):
super(RealDelegate, self).__init__()
self.dryrun = dryrun
self.mailfrom = mailfrom
self.mailto = mailto
if gsutil:
self.gsutil = gsutil
else:
self.gsutil = buildbot_common.GetGsutil()
def update_mono_sdk_json(infos):
'''Update the naclmono manifest with the newly built packages'''
if len(infos) == 0:
return
manifest_file = open(MONO_MANIFEST, 'r')
mono_manifest = json.loads(manifest_file.read())
manifest_file.close()
for info in infos:
bundle = {}
bundle['name'] = info['naclmono_name']
bundle['description'] = 'Mono for Native Client'
bundle['stability'] = info['stability']
bundle['recommended'] = 'no'
bundle['version'] = 'experimental'
archive = {}
sha1_hash = hashlib.sha1()
f = open(info['naclmono_name'] + '.bz2', 'rb')
sha1_hash.update(f.read())
archive['size'] = f.tell()
f.close()
archive['checksum'] = {'sha1': sha1_hash.hexdigest()}
archive['host_os'] = 'all'
archive['url'] = ('https://commondatastorage.googleapis.com/'
'nativeclient-mirror/nacl/nacl_sdk/%s/%s/%s.bz2' %
(info['naclmono_name'], info['naclmono_rev'],
info['naclmono_name']))
bundle['archives'] = [archive]
bundle['revision'] = int(info['naclmono_rev'])
bundle['sdk_revision'] = int(info['sdk_revision'])
# Insert this new bundle into the manifest,
# probably overwriting an existing bundle.
for key, value in mono_manifest.items():
if key == 'bundles':
existing = filter(lambda b: b['name'] == info['naclmono_name'],
value)
if len(existing) > 0:
loc = value.index(existing[0])
value[loc] = bundle
else:
value.append(bundle)
# Write out the file locally, then upload to its known location.
manifest_file = open(MONO_MANIFEST, 'w')
manifest_file.write(json.dumps(mono_manifest, sort_keys=False, indent=2))
manifest_file.close()
buildbot_common.Run([
buildbot_common.GetGsutil(), 'cp', '-a', 'public-read', MONO_MANIFEST,
GS_MANIFEST_PATH + MONO_MANIFEST
])
def _RunGsUtil(self, stdin, *args):
"""Run gsutil as a subprocess.
Args:
stdin: If non-None, used as input to the process.
*args: Arguments to pass to gsutil. The first argument should be an
operation such as ls, cp or cat.
Returns:
The stdout from the process."""
cmd = [buildbot_common.GetGsutil()] + list(args)
if stdin:
stdin_pipe = subprocess.PIPE
else:
stdin_pipe = None
process = subprocess.Popen(cmd, stdin=stdin_pipe, stdout=subprocess.PIPE)
stdout, _ = process.communicate(stdin)
if process.returncode != 0:
raise subprocess.CalledProcessError(process.returncode, ' '.join(cmd))
return stdout
def main(args):
parser = optparse.OptionParser()
parser.add_option('--arch',
help='Target architecture',
dest='arch',
default='x86-32')
parser.add_option('--sdk-revision',
help='SDK Revision'
' (default=buildbot revision)',
dest='sdk_revision',
default=None)
parser.add_option('--sdk-url',
help='SDK Download URL',
dest='sdk_url',
default=None)
parser.add_option('--install-dir',
help='Install Directory',
dest='install_dir',
default='naclmono')
(options, args) = parser.parse_args(args[1:])
assert sys.platform.find('linux') != -1
buildbot_revision = os.environ.get('BUILDBOT_REVISION', '')
build_prefix = options.arch + ' '
buildbot_common.BuildStep(build_prefix + 'Clean Old SDK')
buildbot_common.MakeDir(MONO_BUILD_DIR)
buildbot_common.RemoveDir(os.path.join(MONO_BUILD_DIR, 'pepper_*'))
buildbot_common.BuildStep(build_prefix + 'Setup New SDK')
sdk_dir = None
sdk_revision = options.sdk_revision
sdk_url = options.sdk_url
if not sdk_url:
if not sdk_revision:
assert buildbot_revision
sdk_revision = buildbot_revision.split(':')[0]
sdk_url = 'gs://nativeclient-mirror/nacl/nacl_sdk/'\
'trunk.%s/naclsdk_linux.bz2' % sdk_revision
sdk_url = sdk_url.replace('https://commondatastorage.googleapis.com/',
'gs://')
sdk_file = sdk_url.split('/')[-1]
buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', sdk_url, sdk_file],
cwd=MONO_BUILD_DIR)
tar_file = None
try:
tar_file = tarfile.open(os.path.join(MONO_BUILD_DIR, sdk_file))
pepper_dir = os.path.commonprefix(tar_file.getnames())
tar_file.extractall(path=MONO_BUILD_DIR)
sdk_dir = os.path.join(MONO_BUILD_DIR, pepper_dir)
finally:
if tar_file:
tar_file.close()
assert sdk_dir
buildbot_common.BuildStep(build_prefix + 'Checkout Mono')
# TODO(elijahtaylor): Get git URL from master/trigger to make this
# more flexible for building from upstream and release branches.
git_url = 'git://github.com/elijahtaylor/mono.git'
git_rev = 'HEAD'
if buildbot_revision:
git_rev = buildbot_revision.split(':')[1]
if not os.path.exists(MONO_DIR):
buildbot_common.MakeDir(MONO_DIR)
buildbot_common.Run(['git', 'clone', git_url, MONO_DIR])
else:
buildbot_common.Run(['git', 'fetch'], cwd=MONO_DIR)
if git_rev:
buildbot_common.Run(['git', 'checkout', git_rev], cwd=MONO_DIR)
arch_to_bitsize = {'x86-32': '32', 'x86-64': '64'}
arch_to_output_folder = {
'x86-32': 'runtime-build',
'x86-64': 'runtime64-build'
}
buildbot_common.BuildStep(build_prefix + 'Configure Mono')
os.environ['NACL_SDK_ROOT'] = sdk_dir
os.environ['TARGET_BITSIZE'] = arch_to_bitsize[options.arch]
buildbot_common.Run(['./autogen.sh'], cwd=MONO_DIR)
buildbot_common.Run(['make', 'distclean'], cwd=MONO_DIR)
buildbot_common.BuildStep(build_prefix + 'Build and Install Mono')
nacl_interp_script = os.path.join(SDK_BUILD_DIR,
'nacl_interp_loader_mono.sh')
os.environ['NACL_INTERP_LOADER'] = nacl_interp_script
buildbot_common.Run(
[
'./nacl-mono-runtime.sh',
MONO_DIR, # Mono directory with 'configure'
arch_to_output_folder[options.arch], # Build dir
options.install_dir
],
cwd=SDK_BUILD_DIR)
buildbot_common.BuildStep(build_prefix + 'Test Mono')
buildbot_common.Run(['make', 'check', '-j8'],
cwd=os.path.join(SDK_BUILD_DIR,
arch_to_output_folder[options.arch]))
return 0
def get_sdk_build_info():
'''Returns a list of dictionaries for versions of NaCl Mono to build which are
out of date compared to the SDKs available to naclsdk'''
# Get a copy of the naclsdk manifest file
buildbot_common.Run([buildbot_common.GetGsutil(), 'cp',
GS_MANIFEST_PATH + SDK_MANIFEST, '.'])
manifest_file = open(SDK_MANIFEST, 'r')
sdk_manifest = json.loads(manifest_file.read())
manifest_file.close()
pepper_infos = []
for key, value in sdk_manifest.items():
if key == 'bundles':
stabilities = ['stable', 'beta', 'dev', 'post_stable']
# Pick pepper_* bundles, need pepper_19 or greater to build Mono
bundles = filter(lambda b: (b['stability'] in stabilities
and 'pepper_' in b['name'])
and b['version'] >= 19, value)
for b in bundles:
newdict = {}
newdict['pepper_revision'] = str(b['version'])
linux_arch = filter(lambda u: u['host_os'] == 'linux', b['archives'])
newdict['sdk_url'] = linux_arch[0]['url']
newdict['sdk_revision'] = b['revision']
newdict['stability'] = b['stability']
newdict['naclmono_name'] = 'naclmono_' + newdict['pepper_revision']
pepper_infos.append(newdict)
# Get a copy of the naclmono manifest file
buildbot_common.Run([buildbot_common.GetGsutil(), 'cp',
GS_MANIFEST_PATH + MONO_MANIFEST, '.'])
manifest_file = open(MONO_MANIFEST, 'r')
mono_manifest = json.loads(manifest_file.read())
manifest_file.close()
ret = []
mono_manifest_dirty = False
# Check to see if we need to rebuild mono based on sdk revision
for key, value in mono_manifest.items():
if key == 'bundles':
for info in pepper_infos:
bundle = filter(lambda b: b['name'] == info['naclmono_name'], value)
if len(bundle) == 0:
info['naclmono_rev'] = '1'
ret.append(info)
else:
if info['sdk_revision'] != bundle[0]['sdk_revision']:
# This bundle exists in the mono manifest, bump the revision
# for the new build we're about to make.
info['naclmono_rev'] = str(bundle[0]['revision'] + 1)
ret.append(info)
elif info['stability'] != bundle[0]['stability']:
# If all that happened was the SDK bundle was promoted in stability,
# change only that and re-write the manifest
mono_manifest_dirty = True
bundle[0]['stability'] = info['stability']
# re-write the manifest here because there are no bundles to build but
# the manifest has changed
if mono_manifest_dirty and len(ret) == 0:
manifest_file = open(MONO_MANIFEST, 'w')
manifest_file.write(json.dumps(mono_manifest, sort_keys=False, indent=2))
manifest_file.close()
buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', '-a', 'public-read',
MONO_MANIFEST, GS_MANIFEST_PATH + MONO_MANIFEST])
return ret
def main(args):
parser = optparse.OptionParser()
parser.add_option('--arch',
help='Target architecture',
dest='arch',
default='x86-32')
parser.add_option('--sdk-revision',
help='SDK Revision'
' (default=buildbot revision)',
dest='sdk_revision',
default=None)
parser.add_option('--sdk-url',
help='SDK Download URL',
dest='sdk_url',
default=None)
parser.add_option('--install-dir',
help='Install Directory',
dest='install_dir',
default='naclmono')
(options, args) = parser.parse_args(args[1:])
assert sys.platform.find('linux') != -1
buildbot_revision = os.environ.get('BUILDBOT_REVISION', '')
build_prefix = options.arch + ' '
buildbot_common.BuildStep(build_prefix + 'Clean Old SDK')
buildbot_common.MakeDir(MONO_BUILD_DIR)
buildbot_common.RemoveDir(os.path.join(MONO_BUILD_DIR, 'pepper_*'))
buildbot_common.BuildStep(build_prefix + 'Setup New SDK')
sdk_dir = None
sdk_revision = options.sdk_revision
sdk_url = options.sdk_url
if not sdk_url:
if not sdk_revision:
assert buildbot_revision
sdk_revision = buildbot_revision.split(':')[0]
sdk_url = 'gs://nativeclient-mirror/nacl/nacl_sdk/'\
'trunk.%s/naclsdk_linux.tar.bz2' % sdk_revision
sdk_url = sdk_url.replace('https://commondatastorage.googleapis.com/',
'gs://')
sdk_file = sdk_url.split('/')[-1]
buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', sdk_url, sdk_file],
cwd=MONO_BUILD_DIR)
tar_file = None
try:
tar_file = tarfile.open(os.path.join(MONO_BUILD_DIR, sdk_file))
pepper_dir = os.path.commonprefix(tar_file.getnames())
tar_file.extractall(path=MONO_BUILD_DIR)
sdk_dir = os.path.join(MONO_BUILD_DIR, pepper_dir)
finally:
if tar_file:
tar_file.close()
assert sdk_dir
buildbot_common.BuildStep(build_prefix + 'Checkout Mono')
# TODO(elijahtaylor): Get git URL from master/trigger to make this
# more flexible for building from upstream and release branches.
if options.arch == 'arm':
git_url = 'git://github.com/igotti-google/mono.git'
git_rev = 'arm_nacl'
else:
git_url = 'git://github.com/elijahtaylor/mono.git'
git_rev = 'HEAD'
if buildbot_revision:
# Unfortunately, we use different git branches/revisions
# for ARM and x86 now, so ignore buildbot_revision variable for ARM.
# Need to rethink this approach, if we'll plan to support
# more flexible repo selection mechanism.
if options.arch != 'arm':
git_rev = buildbot_revision.split(':')[1]
# ARM and x86 is built out of different git trees, so distinguish
# them by appending the arch. It also makes 32 and 64 bit x86 separated,
# which is good.
# TODO(olonho): maybe we need to avoid modifications of global.
global MONO_DIR
tag = options.arch
MONO_DIR = "%s-%s" % (MONO_DIR, tag)
if not os.path.exists(MONO_DIR):
buildbot_common.MakeDir(MONO_DIR)
buildbot_common.Run(['git', 'clone', git_url, MONO_DIR])
else:
buildbot_common.Run(['git', 'fetch'], cwd=MONO_DIR)
if git_rev:
buildbot_common.Run(['git', 'checkout', git_rev], cwd=MONO_DIR)
arch_to_bitsize = {'x86-32': '32', 'x86-64': '64', 'arm': 'arm'}
arch_to_output_folder = {
'x86-32': 'runtime-x86-32-build',
'x86-64': 'runtime-x86-64-build',
'arm': 'runtime-arm-build'
}
buildbot_common.BuildStep(build_prefix + 'Configure Mono')
os.environ['NACL_SDK_ROOT'] = sdk_dir
os.environ['TARGET_ARCH'] = options.arch
os.environ['TARGET_BITSIZE'] = arch_to_bitsize[options.arch]
buildbot_common.Run(['./autogen.sh'], cwd=MONO_DIR)
buildbot_common.Run(['make', 'distclean'], cwd=MONO_DIR)
buildbot_common.BuildStep(build_prefix + 'Build and Install Mono')
nacl_interp_script = os.path.join(SDK_BUILD_DIR,
'nacl_interp_loader_mono.sh')
os.environ['NACL_INTERP_LOADER'] = nacl_interp_script
buildbot_common.Run(
[
'./nacl-mono-runtime.sh',
MONO_DIR, # Mono directory with 'configure'
arch_to_output_folder[options.arch], # Build dir
options.install_dir
],
cwd=SDK_BUILD_DIR)
# TODO(elijahtaylor,olonho): Re-enable tests on arm when they compile/run.
if options.arch != 'arm':
buildbot_common.BuildStep(build_prefix + 'Test Mono')
buildbot_common.Run(['make', 'check', '-j8'],
cwd=os.path.join(
SDK_BUILD_DIR,
arch_to_output_folder[options.arch]))
return 0
def __init__(self, dryrun=False, gsutil=None):
self.dryrun = dryrun
if gsutil:
self.gsutil = gsutil
else:
self.gsutil = buildbot_common.GetGsutil()
