def main(args):
scriptlog = setup_logging(args)
logging.info("Starting %s build" % BUILD_TYPE)
# for total runtime
script_start = datetime.now()
logging.info(script_start)
# set vars for uploading/mirroring
if not args.user:
droid_user = os.getenv('DROID_USER')
else:
droid_user = args.user
if not args.host:
droid_host = os.getenv('DROID_HOST')
else:
droid_host = args.host
if not args.remotedir:
droid_path = os.getenv('DROID_PATH')
else:
droid_path= args.remotedir
if not args.localdir:
droid_mirror = os.getenv('DROID_MIRROR')
if not droid_mirror:
droid_mirror = os.getenv('DROID_LOCAL_MIRROR')
else:
droid_mirror = args.localdir
if not args.port:
droid_host_port = os.getenv('DROID_HOST_PORT')
if not droid_host_port:
droid_host_port = '22'
else:
droid_host_port = args.port
uploading = False
if droid_host and droid_user and droid_path:
uploading = True
mirroring = False
if droid_mirror:
mirroring = True
# we must put the builds somewhere
if not mirroring and not uploading:
logging.error("DROID_MIRROR not set")
logging.error("DROID_HOST or DROID_USER or DROID_PATH not set")
logging.error("no where put builds. BAILING!!")
exit()
# cd working dir
previous_working_dir = os.getcwd()
os.chdir(args.source)
if uploading:
upload_path = droid_path
# upload thread
upq = Queue.Queue()
t1 = rsync.rsyncThread(upq, port=droid_host_port, message='Uploaded')
t1.setDaemon(True)
t1.start()
if mirroring:
mirror_path = droid_mirror
# mirror thread
m_q = Queue.Queue()
t2 = rsync.rsyncThread(m_q, message='Mirrored')
t2.setDaemon(True)
t2.start()
# Sync
html_changelog = get_changelog(args)
if html_changelog is not None:
# add changelog to rsync queues
if uploading:
# make the remote directories
full_upload_path_cl = os.path.join(upload_path, get_codename())
try:
subprocess.check_call(('ssh',
'-p%s' % (droid_host_port),
'%s@%s' % (droid_user, droid_host),
'test -d %s || mkdir -p %s' %
(full_upload_path_cl, full_upload_path_cl)
))
except subprocess.CalledProcessError as e:
logging.error('ssh returned %d while making %s' %
(e.returncode, full_upload_path_cl))
else:
upq.put((html_changelog,
"%s@%s:%s" % (droid_user, droid_host, full_upload_path_cl)
))
if mirroring:
full_mirror_path_cl = os.path.join(mirror_path, get_codename())
try:
if not os.path.isdir(full_mirror_path_cl):
os.makedirs(full_mirror_path_cl)
except OSError as e:
logging.error(e)
else:
m_q.put((html_changelog,
full_mirror_path_cl
))
else:
if NIGHTLY_BUILD and not args.force: # No changes, dont do the build
return
#
# Building
#
# for zip storage
if os.path.isdir("/dev/shm"):
temp_dir = "/dev/shm/tmp-%s-droidbuilder_zips" % os.getenv("USER")
else:
temp_dir = "/tmp/tmp-%s-droidbuilder_zips" % os.getenv("USER")
if not os.path.isdir(temp_dir):
os.makedirs(temp_dir)
# keep track of builds
build_start = datetime.now()
# for json manifest
json_info = []
# Targets to run through the squisher
squisher_targets = (
'passion',
'bravo',
'supersonic',
'inc',
)
# Export the proper build type
if TESTING_BUILD:
os.putenv('TESTING_BUILD','true')
os.putenv('RELEASE_BUILD','false')
os.putenv('NIGHTLY_BUILD','false')
elif RELEASE_BUILD:
os.putenv('TESTING_BUILD','false')
os.putenv('RELEASE_BUILD','true')
os.putenv('NIGHTLY_BUILD','false')
elif NIGHTLY_BUILD:
os.putenv('TESTING_BUILD','false')
os.putenv('RELEASE_BUILD','false')
os.putenv('NIGHTLY_BUILD','true')
# build each target
for target in args.target:
if not args.nobuild:
target_start = datetime.now()
pkg = 'otapackage'
if target in squisher_targets:
pkg = 'squishedpackage'
if android.build(target, pkg, not args.rebuild):
continue # Failed #TODO reverse return value
else:
logging.info('Built %s in %s' %
(target, pretty_time(datetime.now() - target_start)))
# find and add the zips to the rsync queues
zips = []
target_out_dir = os.path.join('out', 'target', 'product', target)
if os.path.isdir(target_out_dir):
for f in os.listdir(target_out_dir):
if f.startswith('ev') and f.endswith('.zip'):
zips.append(f)
if zips:
# testing and release builds are put in codename subdirs
# nightly will just be DATE
codename = get_codename(target)
if codename:
if uploading:
# make the remote directories
full_upload_path = os.path.join(upload_path, codename)
try:
subprocess.check_call(('ssh',
'-p%s' % (droid_host_port),
'%s@%s' % (droid_user, droid_host),
'test -d %s || mkdir -p %s' %
(full_upload_path, full_upload_path)
))
except subprocess.CalledProcessError as e:
logging.error('ssh returned %d while making %s' %
(e.returncode, full_upload_path))
if not mirroring:
continue
if mirroring:
full_mirror_path = os.path.join(mirror_path, codename)
try:
if not os.path.isdir(full_mirror_path):
os.makedirs(full_mirror_path)
except OSError as e:
logging.error(e)
continue
zip_info = []
for z in zips:
zip_path = os.path.join(target_out_dir, z)
temp_zip_path = os.path.join(temp_dir, z)
zip_info.append({
'date': DATE,
'device': target,
'count': 0,
'message': get_message(target,args),
'md5sum': md5sum(zip_path),
'name': z,
'size': os.path.getsize(zip_path),
'type': BUILD_TYPE,
'location': '%s/%s' % (codename,z),
})
shutil.copy2(zip_path, temp_zip_path)
if uploading:
upq.put((temp_zip_path,
'%s@%s:%s' % (droid_user, droid_host, full_upload_path)
))
if mirroring:
m_q.put((temp_zip_path,
full_mirror_path
))
json_info.append({
'codename': codename,
'zip_info': zip_info,
})
else:
logging.error('Failed to get codename for %s' % (target))
else:
logging.warning('No zips found for %s' % target)
# write total buildtime
logging.info('Built all targets in %s' %
(pretty_time(datetime.now() - build_start)))
# wait for builds to finish uploading/mirroring
if mirroring:
m_q.join()
if uploading:
upq.join()
if json_info and mirroring: # TODO uploading
for entries in json_info:
# Release and testing builds are stored in the same directory,
# so we have to read the old entries, add new ones, and rewrite
# Nightly wont have one initially and all builds are
# stored in the same directory so it will just re(read/write)
# for all builds which is stupid but will work fine
device_manifest = os.path.join(mirror_path,
entries.get('codename'), 'info.json')
device_entries = []
try:
f = open(device_manifest)
except IOError as e:
pass #logging.error('%s' % e)
else:
with f:
device_entries = json.load(f)
for e in entries.get('zip_info'):
device_entries.append(e)
try:
f = open(device_manifest,'w')
except IOError as e:
logging.error(e)
else:
with f:
json.dump(device_entries, f, indent=2)
# cleanup
shutil.rmtree(temp_dir)
logging.info('Total run time: %s' %
(pretty_time(datetime.now() - script_start)))
# Write scriptlog for website
html_scriptlog = write_html_scriptlog(scriptlog)
if html_scriptlog is not None:
# add log to rsync queues
if uploading:
upq.put((html_scriptlog,
"%s@%s:%s" % (droid_user, droid_host, os.path.join(upload_path, get_codename()))
))
upq.join()
if mirroring:
m_q.put((html_scriptlog,
os.path.join(mirror_path, get_codename())
))
m_q.join()
# cd previous working dir
os.chdir(previous_working_dir)
def main(args):
# script logging
log_dir = os.path.join(args.source, 'nightly_logs')
try:
if not os.path.isdir(log_dir):
os.mkdir(log_dir)
except OSError:
pass
scriptlog = os.path.join(log_dir, 'scriptlog-' + DATE + '.log')
logging.basicConfig(filename=scriptlog, level=logging.INFO,
format='%(levelname)-8s %(message)s')
# for total runtime
script_start = datetime.now()
logging.info(script_start)
# set vars for uploading/mirroring
if not args.user:
droid_user = os.getenv('DROID_USER')
else:
droid_user = args.user
if not args.host:
droid_host = os.getenv('DROID_HOST')
else:
droid_host = args.host
if not args.remotedir:
droid_path = os.getenv('DROID_PATH')
else:
droid_path = args.remotedir
if not args.localdir:
droid_mirror = os.getenv('DROID_MIRROR')
if not droid_mirror:
droid_mirror = os.getenv('DROID_LOCAL_MIRROR')
else:
droid_mirror = args.localdir
# we must put the builds somewhere
if not droid_mirror:
mirroring = False
if droid_host and droid_user and droid_path:
uploading = True
else:
logging.error('DROID_MIRROR not set')
logging.error('DROID_HOST or DROID_USER or DROID_PATH not set')
logging.error('no where put builds. BAILING!!')
exit()
else:
mirroring = True
if droid_host and droid_user and droid_path:
uploading = True
else:
uploading = False
# cd working dir
previous_working_dir = os.getcwd()
os.chdir(args.source)
# we want group write
os.umask(002)
# make the remote directories
if uploading:
upload_path = os.path.join(droid_path, DATE)
try:
subprocess.check_call(['ssh', '%s@%s' % (droid_user, droid_host),
'test -d %s || mkdir -p %s' % (upload_path,upload_path)])
except subprocess.CalledProcessError as e:
logging.error('ssh returned %d while making directories' %
(e.returncode))
uploading = False
if not mirroring:
logging.error('no where to put builds. BAILING!!')
exit()
else:
# upload thread
upq = Queue.Queue()
t1 = rsync.rsyncThread(upq,
'%s@%s:%s' % (droid_user, droid_host, upload_path),
message='Uploaded')
t1.setDaemon(True)
t1.start()
if mirroring:
mirror_path = os.path.join(droid_mirror, DATE)
try:
if not os.path.isdir(mirror_path):
os.makedirs(mirror_path)
subprocess.call(['chmod','775','%s' % (mirror_path)])
except OSError as e:
logging.error('failed to make mirror dir: %s' % (e))
mirroring = False
if not uploading:
logging.error('no where to put builds. BAILING!!')
exit()
else:
# mirror thread
m_q = Queue.Queue()
t2 = rsync.rsyncThread(m_q,
mirror_path,
message='Copied')
t2.setDaemon(True)
t2.start()
#
# Syncing
#
if not args.nosync:
# common directory for all changelogs
changelog_dir = os.path.join(os.path.realpath(os.getcwd()), 'nightly_changelogs')
try:
if not os.path.isdir(changelog_dir):
os.mkdir(changelog_dir)
except OSError:
pass
# changelog
changelog = os.path.join(changelog_dir, 'changelog-' + DATE + '.log')
# sync the tree
if android.reposync():
logging.error('Sync failed. Skipping the build')
args.nobuild = True
# Remove out so we dont upload yesterdays build
if os.path.isdir('out'):
shutil.rmtree('out')
else:
android.get_changelog(DATE,changelog)
# create the html changelog
if os.path.exists(changelog):
logging.info('Created changelog for %s' % DATE)
html_changelog = os.path.join(changelog_dir, 'changelog-' + DATE + '.html')
cl = html.Create()
cl.title('Changelog')
cl.css('body {font-family:"Lucida Console", Monaco, monospace;font-size:0.9em;}')
clbody = html.parse_file(changelog)
cl.header(clbody[0])
cl.body(html.add_line_breaks(clbody[1:]))
cl.write(html_changelog)
# add changelog to rsync queues
if uploading:
upq.put(html_changelog)
if mirroring:
m_q.put(html_changelog)
else:
logging.info('Skipped sync')
#
# Building
#
# export vars for the build script
os.putenv('NIGHTLY_BUILD', 'true')
# for zip storage
if os.path.isdir('/dev/shm'):
temp_dir = '/dev/shm/tmp-nightlybuilder_zips'
else:
temp_dir = '/tmp/tmp-nightlybuilder_zips'
if not os.path.isdir(temp_dir):
os.mkdir(temp_dir)
# keep track of builds
build_start = datetime.now()
# for json manifest
json_info = []
# Targets to run through the squisher
squisher_targets = (
'passion',
'bravo',
'supersonic',
'inc',
)
# build each target
for target in args.target:
if not args.nobuild:
target_start = datetime.now()
pkg = 'otapackage'
if target in squisher_targets:
pkg = 'squishedpackage'
if android.build(target,pkg):
continue # Failed
else:
logging.info('Built %s in %s' %
(target, pretty_time(datetime.now() - target_start)))
# find and add the zips to the rsync queues
zips = []
target_out_dir = os.path.join('out', 'target', 'product', target)
if os.path.isdir(target_out_dir):
for f in os.listdir(target_out_dir):
if f.startswith('ev') and f.endswith('.zip'):
zips.append(f)
if zips:
for z in zips:
json_info.append({
'date': DATE,
'device': target,
'count': 0,
'message': 'Nightly build for %s' % target,
'md5sum': md5sum(os.path.join(target_out_dir, z)),
'name': z,
'size': os.path.getsize(os.path.join(target_out_dir, z)),
'type': 'nightly',
'location': '%s/%s' % (DATE,z),
})
shutil.copy2(os.path.join(target_out_dir, z),os.path.join(temp_dir, z))
if uploading:
upq.put(os.path.join(temp_dir, z))
if mirroring:
m_q.put(os.path.join(temp_dir, z))
else:
logging.warning('No zips found for %s' % target)
# write total buildtime
logging.info('Built all targets in %s' %
(pretty_time(datetime.now() - build_start)))
# write manifest
if json_info:
json_info.sort(key=lambda d:d['device'])
try:
f = open(os.path.join(temp_dir,'info.json'),'w')
except IOError as e:
logging.error('Failed to open info.json: %s' % (e))
else:
with f:
json.dump(json_info, f, indent=2)
if uploading:
upq.put(os.path.join(temp_dir,'info.json'))
if mirroring:
m_q.put(os.path.join(temp_dir,'info.json'))
# for website
''' # Not using this any more
if mirroring:
main_manifest = os.path.join(droid_mirror,'manifest.json')
try:
f = open(main_manifest,'r')
except IOError as e:
logging.warning('%s' % e)
else:
with f:
entries = json.load(f)
for e in json_info:
entries.append(e)
try:
f = open(main_manifest,'w')
except IOError as e:
logging.warning('%s' % e)
else:
with f:
json.dump(entries,f,indent=2)
'''
# wait for builds to finish uploading/mirroring
if mirroring:
m_q.join()
if uploading:
upq.join()
# cleanup
shutil.rmtree(temp_dir)
logging.info('Total run time: %s' %
(pretty_time(datetime.now() - script_start)))
#
# Finish up
#
# create html scriptlog
if os.path.exists(scriptlog):
html_scriptlog = os.path.join(log_dir, 'scriptlog-' + DATE + '.html')
sl = html.Create()
sl.title('Nightly Log')
sl.css('body {font-family:"Lucida Console", Monaco, monospace;font-size:0.9em;}')
sl.header(DATE)
sl.body(html.add_line_breaks(html.parse_file(scriptlog)))
sl.write(html_scriptlog)
# add log to rsync queues
if uploading:
upq.put(html_scriptlog)
upq.join()
if mirroring:
m_q.put(html_scriptlog)
m_q.join()
# cd previous working dir
os.chdir(previous_working_dir)
