def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir,
extlib_dir, tmp_dir, force, onserver):
"""Do a build locally."""
if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
if not config['ndk_path']:
logging.critical("$ANDROID_NDK is not set!")
sys.exit(3)
elif not os.path.isdir(config['sdk_path']):
logging.critical(
"$ANDROID_NDK points to a non-existing directory!")
sys.exit(3)
# Prepare the source code...
root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild,
build_dir, srclib_dir,
extlib_dir, onserver)
# We need to clean via the build tool in case the binary dirs are
# different from the default ones
p = None
if thisbuild['type'] == 'maven':
logging.info("Cleaning Maven project...")
cmd = [
config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']
]
if '@' in thisbuild['maven']:
maven_dir = os.path.join(root_dir,
thisbuild['maven'].split('@', 1)[1])
maven_dir = os.path.normpath(maven_dir)
else:
maven_dir = root_dir
p = FDroidPopen(cmd, cwd=maven_dir)
elif thisbuild['type'] == 'gradle':
logging.info("Cleaning Gradle project...")
cmd = [config['gradle'], 'clean']
adapt_gradle(build_dir)
for name, number, libpath in srclibpaths:
adapt_gradle(libpath)
p = FDroidPopen(cmd, cwd=root_dir)
elif thisbuild['type'] == 'kivy':
pass
elif thisbuild['type'] == 'ant':
logging.info("Cleaning Ant project...")
p = FDroidPopen(['ant', 'clean'], cwd=root_dir)
if p is not None and p.returncode != 0:
raise BuildException(
"Error cleaning %s:%s" % (app['id'], thisbuild['version']),
p.output)
for root, dirs, files in os.walk(build_dir):
# Don't remove possibly necessary 'gradle' dirs if 'gradlew' is not there
if 'gradlew' in files:
logging.debug("Getting rid of Gradle wrapper stuff in %s" % root)
os.remove(os.path.join(root, 'gradlew'))
if 'gradlew.bat' in files:
os.remove(os.path.join(root, 'gradlew.bat'))
if 'gradle' in dirs:
shutil.rmtree(os.path.join(root, 'gradle'))
if not options.skipscan:
# Scan before building...
logging.info("Scanning source for common problems...")
count = common.scan_source(build_dir, root_dir, thisbuild)
if count > 0:
if force:
logging.warn('Scanner found %d problems:' % count)
else:
raise BuildException(
"Can't build due to %d errors while scanning" % count)
if not options.notarball:
# Build the source tarball right before we build the release...
logging.info("Creating source tarball...")
tarname = common.getsrcname(app, thisbuild)
tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz")
def tarexc(f):
return any(f.endswith(s) for s in ['.svn', '.git', '.hg', '.bzr'])
tarball.add(build_dir, tarname, exclude=tarexc)
tarball.close()
if onserver:
manifest = os.path.join(root_dir, 'AndroidManifest.xml')
if os.path.exists(manifest):
homedir = os.path.expanduser('~')
with open(os.path.join(homedir, 'buildserverid'), 'r') as f:
buildserverid = f.read()
with open(os.path.join(homedir, 'fdroidserverid'), 'r') as f:
fdroidserverid = f.read()
with open(manifest, 'r') as f:
manifestcontent = f.read()
manifestcontent = manifestcontent.replace(
'</manifest>',
'<fdroid buildserverid="' + buildserverid + '"' +
' fdroidserverid="' + fdroidserverid + '"' + '/></manifest>')
with open(manifest, 'w') as f:
f.write(manifestcontent)
# Run a build command if one is required...
if thisbuild['build']:
logging.info("Running 'build' commands in %s" % root_dir)
cmd = common.replace_config_vars(thisbuild['build'])
# Substitute source library paths into commands...
for name, number, libpath in srclibpaths:
libpath = os.path.relpath(libpath, root_dir)
cmd = cmd.replace('$$' + name + '$$', libpath)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
raise BuildException(
"Error running build command for %s:%s" %
(app['id'], thisbuild['version']), p.output)
# Build native stuff if required...
if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
logging.info("Building the native code")
jni_components = thisbuild['buildjni']
if jni_components == ['yes']:
jni_components = ['']
cmd = [os.path.join(config['ndk_path'], "ndk-build"), "-j1"]
for d in jni_components:
if d:
logging.info("Building native code in '%s'" % d)
else:
logging.info("Building native code in the main project")
manifest = root_dir + '/' + d + '/AndroidManifest.xml'
if os.path.exists(manifest):
# Read and write the whole AM.xml to fix newlines and avoid
# the ndk r8c or later 'wordlist' errors. The outcome of this
# under gnu/linux is the same as when using tools like
# dos2unix, but the native python way is faster and will
# work in non-unix systems.
manifest_text = open(manifest, 'U').read()
open(manifest, 'w').write(manifest_text)
# In case the AM.xml read was big, free the memory
del manifest_text
p = FDroidPopen(cmd, cwd=os.path.join(root_dir, d))
if p.returncode != 0:
raise BuildException(
"NDK build failed for %s:%s" %
(app['id'], thisbuild['version']), p.output)
p = None
# Build the release...
if thisbuild['type'] == 'maven':
logging.info("Building Maven project...")
if '@' in thisbuild['maven']:
maven_dir = os.path.join(root_dir,
thisbuild['maven'].split('@', 1)[1])
else:
maven_dir = root_dir
mvncmd = [
config['mvn3'], '-Dandroid.sdk.path=' + config['sdk_path'],
'-Dmaven.jar.sign.skip=true', '-Dmaven.test.skip=true',
'-Dandroid.sign.debug=false', '-Dandroid.release=true', 'package'
]
if thisbuild['target']:
target = thisbuild["target"].split('-')[1]
FDroidPopen([
'sed', '-i', 's@<platform>[0-9]*</platform>@<platform>' +
target + '</platform>@g', 'pom.xml'
],
cwd=root_dir)
if '@' in thisbuild['maven']:
FDroidPopen([
'sed', '-i', 's@<platform>[0-9]*</platform>@<platform>' +
target + '</platform>@g', 'pom.xml'
],
cwd=maven_dir)
p = FDroidPopen(mvncmd, cwd=maven_dir)
bindir = os.path.join(root_dir, 'target')
elif thisbuild['type'] == 'kivy':
logging.info("Building Kivy project...")
spec = os.path.join(root_dir, 'buildozer.spec')
if not os.path.exists(spec):
raise BuildException(
"Expected to find buildozer-compatible spec at {0}".format(
spec))
defaults = {
'orientation': 'landscape',
'icon': '',
'permissions': '',
'android.api': "18"
}
bconfig = ConfigParser(defaults, allow_no_value=True)
bconfig.read(spec)
distdir = 'python-for-android/dist/fdroid'
if os.path.exists(distdir):
shutil.rmtree(distdir)
modules = bconfig.get('app', 'requirements').split(',')
cmd = 'ANDROIDSDK=' + config['sdk_path']
cmd += ' ANDROIDNDK=' + config['ndk_path']
cmd += ' ANDROIDNDKVER=r9'
cmd += ' ANDROIDAPI=' + str(bconfig.get('app', 'android.api'))
cmd += ' VIRTUALENV=virtualenv'
cmd += ' ./distribute.sh'
cmd += ' -m ' + "'" + ' '.join(modules) + "'"
cmd += ' -d fdroid'
p = FDroidPopen(cmd, cwd='python-for-android', shell=True)
if p.returncode != 0:
raise BuildException("Distribute build failed")
cid = bconfig.get('app', 'package.domain') + '.' + bconfig.get(
'app', 'package.name')
if cid != app['id']:
raise BuildException(
"Package ID mismatch between metadata and spec")
orientation = bconfig.get('app', 'orientation', 'landscape')
if orientation == 'all':
orientation = 'sensor'
cmd = [
'./build.py'
'--dir', root_dir, '--name',
bconfig.get('app', 'title'), '--package', app['id'], '--version',
bconfig.get('app', 'version'), '--orientation', orientation
]
perms = bconfig.get('app', 'permissions')
for perm in perms.split(','):
cmd.extend(['--permission', perm])
if config.get('app', 'fullscreen') == 0:
cmd.append('--window')
icon = bconfig.get('app', 'icon.filename')
if icon:
cmd.extend(['--icon', os.path.join(root_dir, icon)])
cmd.append('release')
p = FDroidPopen(cmd, cwd=distdir)
elif thisbuild['type'] == 'gradle':
logging.info("Building Gradle project...")
flavours = thisbuild['gradle'].split(',')
if len(flavours) == 1 and flavours[0] in ['main', 'yes', '']:
flavours[0] = ''
commands = [config['gradle']]
if thisbuild['preassemble']:
commands += thisbuild['preassemble'].split()
flavours_cmd = ''.join(flavours)
if flavours_cmd:
flavours_cmd = flavours_cmd[0].upper() + flavours_cmd[1:]
commands += ['assemble' + flavours_cmd + 'Release']
# Avoid having to use lintOptions.abortOnError false
if thisbuild['gradlepluginver'] >= LooseVersion('0.7'):
with open(os.path.join(root_dir, 'build.gradle'), "a") as f:
f.write(
"\nandroid { lintOptions { checkReleaseBuilds false } }\n")
p = FDroidPopen(commands, cwd=root_dir)
elif thisbuild['type'] == 'ant':
logging.info("Building Ant project...")
cmd = ['ant']
if thisbuild['antcommand']:
cmd += [thisbuild['antcommand']]
else:
cmd += ['release']
p = FDroidPopen(cmd, cwd=root_dir)
bindir = os.path.join(root_dir, 'bin')
if p is not None and p.returncode != 0:
raise BuildException(
"Build failed for %s:%s" % (app['id'], thisbuild['version']),
p.output)
logging.info("Successfully built version " + thisbuild['version'] +
' of ' + app['id'])
if thisbuild['type'] == 'maven':
stdout_apk = '\n'.join([
line for line in p.output.splitlines()
if any(a in line for a in ('.apk', '.ap_', '.jar'))
])
m = re.match(r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", stdout_apk,
re.S | re.M)
if not m:
m = re.match(
r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]",
stdout_apk, re.S | re.M)
if not m:
m = re.match(
r'.*^\[INFO\] [^$]*aapt \[package,[^$]*' + bindir +
r'/([^/]+)\.ap[_k][,\]]', stdout_apk, re.S | re.M)
if not m:
m = re.match(
r".*^\[INFO\] Building jar: .*/" + bindir + r"/(.+)\.jar",
stdout_apk, re.S | re.M)
if not m:
raise BuildException('Failed to find output')
src = m.group(1)
src = os.path.join(bindir, src) + '.apk'
elif thisbuild['type'] == 'kivy':
src = 'python-for-android/dist/default/bin/{0}-{1}-release.apk'.format(
bconfig.get('app', 'title'), bconfig.get('app', 'version'))
elif thisbuild['type'] == 'gradle':
if thisbuild['gradlepluginver'] >= LooseVersion('0.11'):
apks_dir = os.path.join(root_dir, 'build', 'outputs', 'apk')
else:
apks_dir = os.path.join(root_dir, 'build', 'apk')
apks = glob.glob(os.path.join(apks_dir, '*-release-unsigned.apk'))
if len(apks) > 1:
raise BuildException(
'More than one resulting apks found in %s' % apks_dir,
'\n'.join(apks))
if len(apks) < 1:
raise BuildException('Failed to find gradle output in %s' %
apks_dir)
src = apks[0]
elif thisbuild['type'] == 'ant':
stdout_apk = '\n'.join(
[line for line in p.output.splitlines() if '.apk' in line])
src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk,
re.S | re.M).group(1)
src = os.path.join(bindir, src)
elif thisbuild['type'] == 'raw':
src = os.path.join(root_dir, thisbuild['output'])
src = os.path.normpath(src)
# Make sure it's not debuggable...
if common.isApkDebuggable(src, config):
raise BuildException("APK is debuggable")
# By way of a sanity check, make sure the version and version
# code in our new apk match what we expect...
logging.debug("Checking " + src)
if not os.path.exists(src):
raise BuildException("Unsigned apk is not at expected location of " +
src)
p = SilentPopen([config['aapt'], 'dump', 'badging', src])
vercode = None
version = None
foundid = None
nativecode = None
for line in p.output.splitlines():
if line.startswith("package:"):
pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
m = pat.match(line)
if m:
foundid = m.group(1)
pat = re.compile(".*versionCode='([0-9]*)'.*")
m = pat.match(line)
if m:
vercode = m.group(1)
pat = re.compile(".*versionName='([^']*)'.*")
m = pat.match(line)
if m:
version = m.group(1)
elif line.startswith("native-code:"):
nativecode = line[12:]
# Ignore empty strings or any kind of space/newline chars that we don't
# care about
if nativecode is not None:
nativecode = nativecode.strip()
nativecode = None if not nativecode else nativecode
if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
if nativecode is None:
raise BuildException(
"Native code should have been built but none was packaged")
if thisbuild['novcheck']:
vercode = thisbuild['vercode']
version = thisbuild['version']
if not version or not vercode:
raise BuildException(
"Could not find version information in build in output")
if not foundid:
raise BuildException("Could not find package ID in output")
if foundid != app['id']:
raise BuildException("Wrong package ID - build " + foundid +
" but expected " + app['id'])
# Some apps (e.g. Timeriffic) have had the bonkers idea of
# including the entire changelog in the version number. Remove
# it so we can compare. (TODO: might be better to remove it
# before we compile, in fact)
index = version.find(" //")
if index != -1:
version = version[:index]
if (version != thisbuild['version'] or vercode != thisbuild['vercode']):
raise BuildException(("Unexpected version/version code in output;"
" APK: '%s' / '%s', "
" Expected: '%s' / '%s'") %
(version, str(vercode), thisbuild['version'],
str(thisbuild['vercode'])))
# Copy the unsigned apk to our destination directory for further
# processing (by publish.py)...
dest = os.path.join(output_dir, common.getapkname(app, thisbuild))
shutil.copyfile(src, dest)
# Move the source tarball into the output directory...
if output_dir != tmp_dir and not options.notarball:
shutil.move(os.path.join(tmp_dir, tarname),
os.path.join(output_dir, tarname))
def build_server(app, thisbuild, vcs, build_dir, output_dir, force):
"""Do a build on the build server."""
try:
paramiko
except NameError:
raise BuildException("Paramiko is required to use the buildserver")
if options.verbose:
logging.getLogger("paramiko").setLevel(logging.DEBUG)
else:
logging.getLogger("paramiko").setLevel(logging.WARN)
sshinfo = get_clean_vm()
try:
# Open SSH connection...
logging.info("Connecting to virtual machine...")
sshs = paramiko.SSHClient()
sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy())
sshs.connect(sshinfo['hostname'],
username=sshinfo['user'],
port=sshinfo['port'],
timeout=300,
look_for_keys=False,
key_filename=sshinfo['idfile'])
homedir = '/home/' + sshinfo['user']
# Get an SFTP connection...
ftp = sshs.open_sftp()
ftp.get_channel().settimeout(15)
# Put all the necessary files in place...
ftp.chdir(homedir)
# Helper to copy the contents of a directory to the server...
def send_dir(path):
root = os.path.dirname(path)
main = os.path.basename(path)
ftp.mkdir(main)
for r, d, f in os.walk(path):
rr = os.path.relpath(r, root)
ftp.chdir(rr)
for dd in d:
ftp.mkdir(dd)
for ff in f:
lfile = os.path.join(root, rr, ff)
if not os.path.islink(lfile):
ftp.put(lfile, ff)
ftp.chmod(ff, os.stat(lfile).st_mode)
for i in range(len(rr.split('/'))):
ftp.chdir('..')
ftp.chdir('..')
logging.info("Preparing server for build...")
serverpath = os.path.abspath(os.path.dirname(__file__))
ftp.put(os.path.join(serverpath, 'build.py'), 'build.py')
ftp.put(os.path.join(serverpath, 'common.py'), 'common.py')
ftp.put(os.path.join(serverpath, 'metadata.py'), 'metadata.py')
ftp.put(
os.path.join(serverpath, '..', 'buildserver',
'config.buildserver.py'), 'config.py')
ftp.chmod('config.py', 0o600)
# Copy over the ID (head commit hash) of the fdroidserver in use...
subprocess.call('git rev-parse HEAD >' +
os.path.join(os.getcwd(), 'tmp', 'fdroidserverid'),
shell=True,
cwd=serverpath)
ftp.put('tmp/fdroidserverid', 'fdroidserverid')
# Copy the metadata - just the file for this app...
ftp.mkdir('metadata')
ftp.mkdir('srclibs')
ftp.chdir('metadata')
ftp.put(os.path.join('metadata', app['id'] + '.txt'),
app['id'] + '.txt')
# And patches if there are any...
if os.path.exists(os.path.join('metadata', app['id'])):
send_dir(os.path.join('metadata', app['id']))
ftp.chdir(homedir)
# Create the build directory...
ftp.mkdir('build')
ftp.chdir('build')
ftp.mkdir('extlib')
ftp.mkdir('srclib')
# Copy any extlibs that are required...
if thisbuild['extlibs']:
ftp.chdir(homedir + '/build/extlib')
for lib in thisbuild['extlibs']:
lib = lib.strip()
libsrc = os.path.join('build/extlib', lib)
if not os.path.exists(libsrc):
raise BuildException("Missing extlib {0}".format(libsrc))
lp = lib.split('/')
for d in lp[:-1]:
if d not in ftp.listdir():
ftp.mkdir(d)
ftp.chdir(d)
ftp.put(libsrc, lp[-1])
for _ in lp[:-1]:
ftp.chdir('..')
# Copy any srclibs that are required...
srclibpaths = []
if thisbuild['srclibs']:
for lib in thisbuild['srclibs']:
srclibpaths.append(
common.getsrclib(lib,
'build/srclib',
srclibpaths,
basepath=True,
prepare=False))
# If one was used for the main source, add that too.
basesrclib = vcs.getsrclib()
if basesrclib:
srclibpaths.append(basesrclib)
for name, number, lib in srclibpaths:
logging.info("Sending srclib '%s'" % lib)
ftp.chdir(homedir + '/build/srclib')
if not os.path.exists(lib):
raise BuildException("Missing srclib directory '" + lib + "'")
fv = '.fdroidvcs-' + name
ftp.put(os.path.join('build/srclib', fv), fv)
send_dir(lib)
# Copy the metadata file too...
ftp.chdir(homedir + '/srclibs')
ftp.put(os.path.join('srclibs', name + '.txt'), name + '.txt')
# Copy the main app source code
# (no need if it's a srclib)
if (not basesrclib) and os.path.exists(build_dir):
ftp.chdir(homedir + '/build')
fv = '.fdroidvcs-' + app['id']
ftp.put(os.path.join('build', fv), fv)
send_dir(build_dir)
# Execute the build script...
logging.info("Starting build...")
chan = sshs.get_transport().open_session()
chan.get_pty()
cmdline = 'python build.py --on-server'
if force:
cmdline += ' --force --test'
if options.verbose:
cmdline += ' --verbose'
cmdline += " %s:%s" % (app['id'], thisbuild['vercode'])
chan.exec_command('bash -c ". ~/.bsenv && ' + cmdline + '"')
output = ''
while not chan.exit_status_ready():
while chan.recv_ready():
output += chan.recv(1024)
time.sleep(0.1)
logging.info("...getting exit status")
returncode = chan.recv_exit_status()
while True:
get = chan.recv(1024)
if len(get) == 0:
break
output += get
if returncode != 0:
raise BuildException(
"Build.py failed on server for {0}:{1}".format(
app['id'], thisbuild['version']), output)
# Retrieve the built files...
logging.info("Retrieving build output...")
if force:
ftp.chdir(homedir + '/tmp')
else:
ftp.chdir(homedir + '/unsigned')
apkfile = common.getapkname(app, thisbuild)
tarball = common.getsrcname(app, thisbuild)
try:
ftp.get(apkfile, os.path.join(output_dir, apkfile))
if not options.notarball:
ftp.get(tarball, os.path.join(output_dir, tarball))
except:
raise BuildException(
"Build failed for %s:%s - missing output files".format(
app['id'], thisbuild['version']), output)
ftp.close()
finally:
# Suspend the build server.
release_vm()
def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh):
"""Do a build locally."""
ndk_path = build.ndk_path()
if build.buildjni and build.buildjni != ['no']:
if not ndk_path:
logging.critical("Android NDK version '%s' could not be found!" % build.ndk or 'r10e')
logging.critical("Configured versions:")
for k, v in config['ndk_paths'].iteritems():
if k.endswith("_orig"):
continue
logging.critical(" %s: %s" % (k, v))
sys.exit(3)
elif not os.path.isdir(ndk_path):
logging.critical("Android NDK '%s' is not a directory!" % ndk_path)
sys.exit(3)
# Set up environment vars that depend on each build
for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
common.env[n] = ndk_path
common.reset_env_path()
# Set up the current NDK to the PATH
common.add_to_env_path(ndk_path)
# Prepare the source code...
root_dir, srclibpaths = common.prepare_source(vcs, app, build,
build_dir, srclib_dir,
extlib_dir, onserver, refresh)
# We need to clean via the build tool in case the binary dirs are
# different from the default ones
p = None
gradletasks = []
bmethod = build.build_method()
if bmethod == 'maven':
logging.info("Cleaning Maven project...")
cmd = [config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']]
if '@' in build.maven:
maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1])
maven_dir = os.path.normpath(maven_dir)
else:
maven_dir = root_dir
p = FDroidPopen(cmd, cwd=maven_dir)
elif bmethod == 'gradle':
logging.info("Cleaning Gradle project...")
if build.preassemble:
gradletasks += build.preassemble
flavours = build.gradle
if flavours == ['yes']:
flavours = []
flavours_cmd = ''.join([capitalize_intact(f) for f in flavours])
gradletasks += ['assemble' + flavours_cmd + 'Release']
adapt_gradle(build_dir)
for name, number, libpath in srclibpaths:
adapt_gradle(libpath)
cmd = [config['gradle']]
if build.gradleprops:
cmd += ['-P' + kv for kv in build.gradleprops]
cmd += ['clean']
p = FDroidPopen(cmd, cwd=root_dir)
elif bmethod == 'kivy':
pass
elif bmethod == 'ant':
logging.info("Cleaning Ant project...")
p = FDroidPopen(['ant', 'clean'], cwd=root_dir)
if p is not None and p.returncode != 0:
raise BuildException("Error cleaning %s:%s" %
(app.id, build.version), p.output)
for root, dirs, files in os.walk(build_dir):
def del_dirs(dl):
for d in dl:
if d in dirs:
shutil.rmtree(os.path.join(root, d))
def del_files(fl):
for f in fl:
if f in files:
os.remove(os.path.join(root, f))
if 'build.gradle' in files:
# Even when running clean, gradle stores task/artifact caches in
# .gradle/ as binary files. To avoid overcomplicating the scanner,
# manually delete them, just like `gradle clean` should have removed
# the build/ dirs.
del_dirs(['build', '.gradle'])
del_files(['gradlew', 'gradlew.bat'])
if 'pom.xml' in files:
del_dirs(['target'])
if any(f in files for f in ['ant.properties', 'project.properties', 'build.xml']):
del_dirs(['bin', 'gen'])
if 'jni' in dirs:
del_dirs(['obj'])
if options.skipscan:
if build.scandelete:
raise BuildException("Refusing to skip source scan since scandelete is present")
else:
# Scan before building...
logging.info("Scanning source for common problems...")
count = scanner.scan_source(build_dir, root_dir, build)
if count > 0:
if force:
logging.warn('Scanner found %d problems' % count)
else:
raise BuildException("Can't build due to %d errors while scanning" % count)
if not options.notarball:
# Build the source tarball right before we build the release...
logging.info("Creating source tarball...")
tarname = common.getsrcname(app, build)
tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz")
def tarexc(f):
return any(f.endswith(s) for s in ['.svn', '.git', '.hg', '.bzr'])
tarball.add(build_dir, tarname, exclude=tarexc)
tarball.close()
# Run a build command if one is required...
if build.build:
logging.info("Running 'build' commands in %s" % root_dir)
cmd = common.replace_config_vars(build.build, build)
# Substitute source library paths into commands...
for name, number, libpath in srclibpaths:
libpath = os.path.relpath(libpath, root_dir)
cmd = cmd.replace('$$' + name + '$$', libpath)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
raise BuildException("Error running build command for %s:%s" %
(app.id, build.version), p.output)
# Build native stuff if required...
if build.buildjni and build.buildjni != ['no']:
logging.info("Building the native code")
jni_components = build.buildjni
if jni_components == ['yes']:
jni_components = ['']
cmd = [os.path.join(ndk_path, "ndk-build"), "-j1"]
for d in jni_components:
if d:
logging.info("Building native code in '%s'" % d)
else:
logging.info("Building native code in the main project")
manifest = os.path.join(root_dir, d, 'AndroidManifest.xml')
if os.path.exists(manifest):
# Read and write the whole AM.xml to fix newlines and avoid
# the ndk r8c or later 'wordlist' errors. The outcome of this
# under gnu/linux is the same as when using tools like
# dos2unix, but the native python way is faster and will
# work in non-unix systems.
manifest_text = open(manifest, 'U').read()
open(manifest, 'w').write(manifest_text)
# In case the AM.xml read was big, free the memory
del manifest_text
p = FDroidPopen(cmd, cwd=os.path.join(root_dir, d))
if p.returncode != 0:
raise BuildException("NDK build failed for %s:%s" % (app.id, build.version), p.output)
p = None
# Build the release...
if bmethod == 'maven':
logging.info("Building Maven project...")
if '@' in build.maven:
maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1])
else:
maven_dir = root_dir
mvncmd = [config['mvn3'], '-Dandroid.sdk.path=' + config['sdk_path'],
'-Dmaven.jar.sign.skip=true', '-Dmaven.test.skip=true',
'-Dandroid.sign.debug=false', '-Dandroid.release=true',
'package']
if build.target:
target = build.target.split('-')[1]
common.regsub_file(r'<platform>[0-9]*</platform>',
r'<platform>%s</platform>' % target,
os.path.join(root_dir, 'pom.xml'))
if '@' in build.maven:
common.regsub_file(r'<platform>[0-9]*</platform>',
r'<platform>%s</platform>' % target,
os.path.join(maven_dir, 'pom.xml'))
p = FDroidPopen(mvncmd, cwd=maven_dir)
bindir = os.path.join(root_dir, 'target')
elif bmethod == 'kivy':
logging.info("Building Kivy project...")
spec = os.path.join(root_dir, 'buildozer.spec')
if not os.path.exists(spec):
raise BuildException("Expected to find buildozer-compatible spec at {0}"
.format(spec))
defaults = {'orientation': 'landscape', 'icon': '',
'permissions': '', 'android.api': "18"}
bconfig = ConfigParser(defaults, allow_no_value=True)
bconfig.read(spec)
distdir = os.path.join('python-for-android', 'dist', 'fdroid')
if os.path.exists(distdir):
shutil.rmtree(distdir)
modules = bconfig.get('app', 'requirements').split(',')
cmd = 'ANDROIDSDK=' + config['sdk_path']
cmd += ' ANDROIDNDK=' + ndk_path
cmd += ' ANDROIDNDKVER=' + build.ndk
cmd += ' ANDROIDAPI=' + str(bconfig.get('app', 'android.api'))
cmd += ' VIRTUALENV=virtualenv'
cmd += ' ./distribute.sh'
cmd += ' -m ' + "'" + ' '.join(modules) + "'"
cmd += ' -d fdroid'
p = subprocess.Popen(cmd, cwd='python-for-android', shell=True)
if p.returncode != 0:
raise BuildException("Distribute build failed")
cid = bconfig.get('app', 'package.domain') + '.' + bconfig.get('app', 'package.name')
if cid != app.id:
raise BuildException("Package ID mismatch between metadata and spec")
orientation = bconfig.get('app', 'orientation', 'landscape')
if orientation == 'all':
orientation = 'sensor'
cmd = ['./build.py'
'--dir', root_dir,
'--name', bconfig.get('app', 'title'),
'--package', app.id,
'--version', bconfig.get('app', 'version'),
'--orientation', orientation
]
perms = bconfig.get('app', 'permissions')
for perm in perms.split(','):
cmd.extend(['--permission', perm])
if config.get('app', 'fullscreen') == 0:
cmd.append('--window')
icon = bconfig.get('app', 'icon.filename')
if icon:
cmd.extend(['--icon', os.path.join(root_dir, icon)])
cmd.append('release')
p = FDroidPopen(cmd, cwd=distdir)
elif bmethod == 'gradle':
logging.info("Building Gradle project...")
cmd = [config['gradle']]
if build.gradleprops:
cmd += ['-P' + kv for kv in build.gradleprops]
cmd += gradletasks
p = FDroidPopen(cmd, cwd=root_dir)
elif bmethod == 'ant':
logging.info("Building Ant project...")
cmd = ['ant']
if build.antcommands:
cmd += build.antcommands
else:
cmd += ['release']
p = FDroidPopen(cmd, cwd=root_dir)
bindir = os.path.join(root_dir, 'bin')
if p is not None and p.returncode != 0:
raise BuildException("Build failed for %s:%s" % (app.id, build.version), p.output)
logging.info("Successfully built version " + build.version + ' of ' + app.id)
omethod = build.output_method()
if omethod == 'maven':
stdout_apk = '\n'.join([
line for line in p.output.splitlines() if any(
a in line for a in ('.apk', '.ap_', '.jar'))])
m = re.match(r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk",
stdout_apk, re.S | re.M)
if not m:
m = re.match(r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]",
stdout_apk, re.S | re.M)
if not m:
m = re.match(r'.*^\[INFO\] [^$]*aapt \[package,[^$]*' + bindir + r'/([^/]+)\.ap[_k][,\]]',
stdout_apk, re.S | re.M)
if not m:
m = re.match(r".*^\[INFO\] Building jar: .*/" + bindir + r"/(.+)\.jar",
stdout_apk, re.S | re.M)
if not m:
raise BuildException('Failed to find output')
src = m.group(1)
src = os.path.join(bindir, src) + '.apk'
elif omethod == 'kivy':
src = os.path.join('python-for-android', 'dist', 'default', 'bin',
'{0}-{1}-release.apk'.format(
bconfig.get('app', 'title'),
bconfig.get('app', 'version')))
elif omethod == 'gradle':
src = None
for apks_dir in [
os.path.join(root_dir, 'build', 'outputs', 'apk'),
os.path.join(root_dir, 'build', 'apk'),
]:
for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']:
apks = glob.glob(os.path.join(apks_dir, apkglob))
if len(apks) > 1:
raise BuildException('More than one resulting apks found in %s' % apks_dir,
'\n'.join(apks))
if len(apks) == 1:
src = apks[0]
break
if src is not None:
break
if src is None:
raise BuildException('Failed to find any output apks')
elif omethod == 'ant':
stdout_apk = '\n'.join([
line for line in p.output.splitlines() if '.apk' in line])
src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk,
re.S | re.M).group(1)
src = os.path.join(bindir, src)
elif omethod == 'raw':
globpath = os.path.join(root_dir, build.output)
apks = glob.glob(globpath)
if len(apks) > 1:
raise BuildException('Multiple apks match %s' % globpath, '\n'.join(apks))
if len(apks) < 1:
raise BuildException('No apks match %s' % globpath)
src = os.path.normpath(apks[0])
# Make sure it's not debuggable...
if common.isApkDebuggable(src, config):
raise BuildException("APK is debuggable")
# By way of a sanity check, make sure the version and version
# code in our new apk match what we expect...
logging.debug("Checking " + src)
if not os.path.exists(src):
raise BuildException("Unsigned apk is not at expected location of " + src)
p = SdkToolsPopen(['aapt', 'dump', 'badging', src], output=False)
vercode = None
version = None
foundid = None
nativecode = None
for line in p.output.splitlines():
if line.startswith("package:"):
pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
m = pat.match(line)
if m:
foundid = m.group(1)
pat = re.compile(".*versionCode='([0-9]*)'.*")
m = pat.match(line)
if m:
vercode = m.group(1)
pat = re.compile(".*versionName='([^']*)'.*")
m = pat.match(line)
if m:
version = m.group(1)
elif line.startswith("native-code:"):
nativecode = line[12:]
# Ignore empty strings or any kind of space/newline chars that we don't
# care about
if nativecode is not None:
nativecode = nativecode.strip()
nativecode = None if not nativecode else nativecode
if build.buildjni and build.buildjni != ['no']:
if nativecode is None:
raise BuildException("Native code should have been built but none was packaged")
if build.novcheck:
vercode = build.vercode
version = build.version
if not version or not vercode:
raise BuildException("Could not find version information in build in output")
if not foundid:
raise BuildException("Could not find package ID in output")
if foundid != app.id:
raise BuildException("Wrong package ID - build " + foundid + " but expected " + app.id)
# Some apps (e.g. Timeriffic) have had the bonkers idea of
# including the entire changelog in the version number. Remove
# it so we can compare. (TODO: might be better to remove it
# before we compile, in fact)
index = version.find(" //")
if index != -1:
version = version[:index]
if (version != build.version or
vercode != build.vercode):
raise BuildException(("Unexpected version/version code in output;"
" APK: '%s' / '%s', "
" Expected: '%s' / '%s'")
% (version, str(vercode), build.version,
str(build.vercode))
)
# Add information for 'fdroid verify' to be able to reproduce the build
# environment.
if onserver:
metadir = os.path.join(tmp_dir, 'META-INF')
if not os.path.exists(metadir):
os.mkdir(metadir)
homedir = os.path.expanduser('~')
for fn in ['buildserverid', 'fdroidserverid']:
shutil.copyfile(os.path.join(homedir, fn),
os.path.join(metadir, fn))
subprocess.call(['jar', 'uf', os.path.abspath(src),
'META-INF/' + fn], cwd=tmp_dir)
# Copy the unsigned apk to our destination directory for further
# processing (by publish.py)...
dest = os.path.join(output_dir, common.getapkname(app, build))
shutil.copyfile(src, dest)
# Move the source tarball into the output directory...
if output_dir != tmp_dir and not options.notarball:
shutil.move(os.path.join(tmp_dir, tarname),
os.path.join(output_dir, tarname))
def build_server(app, build, vcs, build_dir, output_dir, force):
"""Do a build on the build server."""
try:
paramiko
except NameError:
raise BuildException("Paramiko is required to use the buildserver")
if options.verbose:
logging.getLogger("paramiko").setLevel(logging.INFO)
else:
logging.getLogger("paramiko").setLevel(logging.WARN)
sshinfo = get_clean_vm()
try:
# Open SSH connection...
logging.info("Connecting to virtual machine...")
sshs = paramiko.SSHClient()
sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy())
sshs.connect(sshinfo['hostname'], username=sshinfo['user'],
port=sshinfo['port'], timeout=300,
look_for_keys=False, key_filename=sshinfo['idfile'])
homedir = '/home/' + sshinfo['user']
# Get an SFTP connection...
ftp = sshs.open_sftp()
ftp.get_channel().settimeout(15)
# Put all the necessary files in place...
ftp.chdir(homedir)
# Helper to copy the contents of a directory to the server...
def send_dir(path):
root = os.path.dirname(path)
main = os.path.basename(path)
ftp.mkdir(main)
for r, d, f in os.walk(path):
rr = os.path.relpath(r, root)
ftp.chdir(rr)
for dd in d:
ftp.mkdir(dd)
for ff in f:
lfile = os.path.join(root, rr, ff)
if not os.path.islink(lfile):
ftp.put(lfile, ff)
ftp.chmod(ff, os.stat(lfile).st_mode)
for i in range(len(rr.split('/'))):
ftp.chdir('..')
ftp.chdir('..')
logging.info("Preparing server for build...")
serverpath = os.path.abspath(os.path.dirname(__file__))
ftp.mkdir('fdroidserver')
ftp.chdir('fdroidserver')
ftp.put(os.path.join(serverpath, '..', 'fdroid'), 'fdroid')
ftp.chmod('fdroid', 0o755)
send_dir(os.path.join(serverpath))
ftp.chdir(homedir)
ftp.put(os.path.join(serverpath, '..', 'buildserver',
'config.buildserver.py'), 'config.py')
ftp.chmod('config.py', 0o600)
# Copy over the ID (head commit hash) of the fdroidserver in use...
subprocess.call('git rev-parse HEAD >' +
os.path.join(os.getcwd(), 'tmp', 'fdroidserverid'),
shell=True, cwd=serverpath)
ftp.put('tmp/fdroidserverid', 'fdroidserverid')
# Copy the metadata - just the file for this app...
ftp.mkdir('metadata')
ftp.mkdir('srclibs')
ftp.chdir('metadata')
ftp.put(os.path.join('metadata', app.id + '.txt'),
app.id + '.txt')
# And patches if there are any...
if os.path.exists(os.path.join('metadata', app.id)):
send_dir(os.path.join('metadata', app.id))
ftp.chdir(homedir)
# Create the build directory...
ftp.mkdir('build')
ftp.chdir('build')
ftp.mkdir('extlib')
ftp.mkdir('srclib')
# Copy any extlibs that are required...
if build.extlibs:
ftp.chdir(homedir + '/build/extlib')
for lib in build.extlibs:
lib = lib.strip()
libsrc = os.path.join('build/extlib', lib)
if not os.path.exists(libsrc):
raise BuildException("Missing extlib {0}".format(libsrc))
lp = lib.split('/')
for d in lp[:-1]:
if d not in ftp.listdir():
ftp.mkdir(d)
ftp.chdir(d)
ftp.put(libsrc, lp[-1])
for _ in lp[:-1]:
ftp.chdir('..')
# Copy any srclibs that are required...
srclibpaths = []
if build.srclibs:
for lib in build.srclibs:
srclibpaths.append(
common.getsrclib(lib, 'build/srclib', basepath=True, prepare=False))
# If one was used for the main source, add that too.
basesrclib = vcs.getsrclib()
if basesrclib:
srclibpaths.append(basesrclib)
for name, number, lib in srclibpaths:
logging.info("Sending srclib '%s'" % lib)
ftp.chdir(homedir + '/build/srclib')
if not os.path.exists(lib):
raise BuildException("Missing srclib directory '" + lib + "'")
fv = '.fdroidvcs-' + name
ftp.put(os.path.join('build/srclib', fv), fv)
send_dir(lib)
# Copy the metadata file too...
ftp.chdir(homedir + '/srclibs')
ftp.put(os.path.join('srclibs', name + '.txt'),
name + '.txt')
# Copy the main app source code
# (no need if it's a srclib)
if (not basesrclib) and os.path.exists(build_dir):
ftp.chdir(homedir + '/build')
fv = '.fdroidvcs-' + app.id
ftp.put(os.path.join('build', fv), fv)
send_dir(build_dir)
# Execute the build script...
logging.info("Starting build...")
chan = sshs.get_transport().open_session()
chan.get_pty()
cmdline = os.path.join(homedir, 'fdroidserver', 'fdroid')
cmdline += ' build --on-server'
if force:
cmdline += ' --force --test'
if options.verbose:
cmdline += ' --verbose'
cmdline += " %s:%s" % (app.id, build.vercode)
chan.exec_command('bash -c ". ~/.bsenv && ' + cmdline + '"')
output = ''
while not chan.exit_status_ready():
while chan.recv_ready():
output += chan.recv(1024)
time.sleep(0.1)
logging.info("...getting exit status")
returncode = chan.recv_exit_status()
while True:
get = chan.recv(1024)
if len(get) == 0:
break
output += get
if returncode != 0:
raise BuildException(
"Build.py failed on server for {0}:{1}".format(
app.id, build.version), output)
# Retrieve the built files...
logging.info("Retrieving build output...")
if force:
ftp.chdir(homedir + '/tmp')
else:
ftp.chdir(homedir + '/unsigned')
apkfile = common.getapkname(app, build)
tarball = common.getsrcname(app, build)
try:
ftp.get(apkfile, os.path.join(output_dir, apkfile))
if not options.notarball:
ftp.get(tarball, os.path.join(output_dir, tarball))
except:
raise BuildException(
"Build failed for %s:%s - missing output files".format(
app.id, build.version), output)
ftp.close()
finally:
# Suspend the build server.
release_vm()
def build_server(app, thisbuild, vcs, build_dir, output_dir, force):
"""Do a build on the build server."""
import ssh
# Reset existing builder machine to a clean state if possible.
vm_ok = False
if not options.resetserver:
print "Checking for valid existing build server"
if got_valid_builder_vm():
print "...VM is present"
p = subprocess.Popen(['VBoxManage', 'snapshot', get_builder_vm_id(), 'list', '--details'],
cwd='builder', stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = p.communicate()[0]
if output.find('fdroidclean') != -1:
if options.verbose:
print "...snapshot exists - resetting build server to clean state"
retcode, output = vagrant(['status'], cwd='builder')
if output.find('running') != -1:
if options.verbose:
print "...suspending"
vagrant(['suspend'], cwd='builder')
p = subprocess.Popen(['VBoxManage', 'snapshot', get_builder_vm_id(), 'restore', 'fdroidclean'],
cwd='builder', stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = p.communicate()[0]
if options.verbose:
print output
if p.returncode == 0:
print "...reset to snapshot - server is valid"
retcode, output = vagrant(['up'], cwd='builder')
if retcode != 0:
raise BuildException("Failed to start build server")
vm_ok = True
else:
print "...failed to reset to snapshot"
else:
print "...snapshot doesn't exist - VBoxManage snapshot list:\n" + output
# If we can't use the existing machine for any reason, make a
# new one from scratch.
if not vm_ok:
if os.path.exists('builder'):
print "Removing broken/incomplete/unwanted build server"
vagrant(['destroy', '-f'], cwd='builder')
shutil.rmtree('builder')
os.mkdir('builder')
p = subprocess.Popen('vagrant --version', shell=True, stdout=subprocess.PIPE)
vver = p.communicate()[0]
if vver.startswith('Vagrant version 1.2'):
with open('builder/Vagrantfile', 'w') as vf:
vf.write('Vagrant.configure("2") do |config|\n')
vf.write('config.vm.box = "buildserver"\n')
vf.write('end\n')
else:
with open('builder/Vagrantfile', 'w') as vf:
vf.write('Vagrant::Config.run do |config|\n')
vf.write('config.vm.box = "buildserver"\n')
vf.write('end\n')
print "Starting new build server"
retcode, _ = vagrant(['up'], cwd='builder')
if retcode != 0:
raise BuildException("Failed to start build server")
# Open SSH connection to make sure it's working and ready...
print "Connecting to virtual machine..."
if subprocess.call('vagrant ssh-config >sshconfig',
cwd='builder', shell=True) != 0:
raise BuildException("Error getting ssh config")
vagranthost = 'default' # Host in ssh config file
sshconfig = ssh.SSHConfig()
sshf = open('builder/sshconfig', 'r')
sshconfig.parse(sshf)
sshf.close()
sshconfig = sshconfig.lookup(vagranthost)
sshs = ssh.SSHClient()
sshs.set_missing_host_key_policy(ssh.AutoAddPolicy())
idfile = sshconfig['identityfile']
if idfile.startswith('"') and idfile.endswith('"'):
idfile = idfile[1:-1]
sshs.connect(sshconfig['hostname'], username=sshconfig['user'],
port=int(sshconfig['port']), timeout=300, look_for_keys=False,
key_filename=idfile)
sshs.close()
print "Saving clean state of new build server"
retcode, _ = vagrant(['suspend'], cwd='builder')
if retcode != 0:
raise BuildException("Failed to suspend build server")
print "...waiting a sec..."
time.sleep(10)
p = subprocess.Popen(['VBoxManage', 'snapshot', get_builder_vm_id(), 'take', 'fdroidclean'],
cwd='builder', stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = p.communicate()[0]
if p.returncode != 0:
print output
raise BuildException("Failed to take snapshot")
print "Restarting new build server"
retcode, _ = vagrant(['up'], cwd='builder')
if retcode != 0:
raise BuildException("Failed to start build server")
# Make sure it worked...
p = subprocess.Popen(['VBoxManage', 'snapshot', get_builder_vm_id(), 'list', '--details'],
cwd='builder', stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = p.communicate()[0]
if output.find('fdroidclean') == -1:
raise BuildException("Failed to take snapshot.")
try:
# Get SSH configuration settings for us to connect...
if options.verbose:
print "Getting ssh configuration..."
subprocess.call('vagrant ssh-config >sshconfig',
cwd='builder', shell=True)
vagranthost = 'default' # Host in ssh config file
# Load and parse the SSH config...
sshconfig = ssh.SSHConfig()
sshf = open('builder/sshconfig', 'r')
sshconfig.parse(sshf)
sshf.close()
sshconfig = sshconfig.lookup(vagranthost)
# Open SSH connection...
if options.verbose:
print "Connecting to virtual machine..."
sshs = ssh.SSHClient()
sshs.set_missing_host_key_policy(ssh.AutoAddPolicy())
idfile = sshconfig['identityfile']
if idfile.startswith('"') and idfile.endswith('"'):
idfile = idfile[1:-1]
sshs.connect(sshconfig['hostname'], username=sshconfig['user'],
port=int(sshconfig['port']), timeout=300, look_for_keys=False,
key_filename=idfile)
# Get an SFTP connection...
ftp = sshs.open_sftp()
ftp.get_channel().settimeout(15)
# Put all the necessary files in place...
ftp.chdir('/home/vagrant')
# Helper to copy the contents of a directory to the server...
def send_dir(path):
root = os.path.dirname(path)
main = os.path.basename(path)
ftp.mkdir(main)
for r, d, f in os.walk(path):
rr = os.path.relpath(r, root)
ftp.chdir(rr)
for dd in d:
ftp.mkdir(dd)
for ff in f:
lfile = os.path.join(root, rr, ff)
if not os.path.islink(lfile):
ftp.put(lfile, ff)
ftp.chmod(ff, os.stat(lfile).st_mode)
for i in range(len(rr.split('/'))):
ftp.chdir('..')
ftp.chdir('..')
print "Preparing server for build..."
serverpath = os.path.abspath(os.path.dirname(__file__))
ftp.put(os.path.join(serverpath, 'build.py'), 'build.py')
ftp.put(os.path.join(serverpath, 'common.py'), 'common.py')
ftp.put(os.path.join(serverpath, 'metadata.py'), 'metadata.py')
ftp.put(os.path.join(serverpath, '..', 'config.buildserver.py'), 'config.py')
ftp.chmod('config.py', 0o600)
# Copy the metadata - just the file for this app...
ftp.mkdir('metadata')
ftp.mkdir('srclibs')
ftp.chdir('metadata')
ftp.put(os.path.join('metadata', app['id'] + '.txt'),
app['id'] + '.txt')
# And patches if there are any...
if os.path.exists(os.path.join('metadata', app['id'])):
send_dir(os.path.join('metadata', app['id']))
ftp.chdir('/home/vagrant')
# Create the build directory...
ftp.mkdir('build')
ftp.chdir('build')
ftp.mkdir('extlib')
ftp.mkdir('srclib')
# Copy any extlibs that are required...
if 'extlibs' in thisbuild:
ftp.chdir('/home/vagrant/build/extlib')
for lib in thisbuild['extlibs'].split(';'):
lib = lib.strip()
libsrc = os.path.join('build/extlib', lib)
if not os.path.exists(libsrc):
raise BuildException("Missing extlib {0}".format(libsrc))
lp = lib.split('/')
for d in lp[:-1]:
if d not in ftp.listdir():
ftp.mkdir(d)
ftp.chdir(d)
ftp.put(libsrc, lp[-1])
for _ in lp[:-1]:
ftp.chdir('..')
# Copy any srclibs that are required...
srclibpaths = []
if 'srclibs' in thisbuild:
for lib in thisbuild['srclibs'].split(';'):
srclibpaths.append(common.getsrclib(lib, 'build/srclib', srclibpaths,
basepath=True, prepare=False))
# If one was used for the main source, add that too.
basesrclib = vcs.getsrclib()
if basesrclib:
srclibpaths.append(basesrclib)
for name, number, lib in srclibpaths:
if options.verbose:
print "Sending srclib '" + lib + "'"
ftp.chdir('/home/vagrant/build/srclib')
if not os.path.exists(lib):
raise BuildException("Missing srclib directory '" + lib + "'")
fv = '.fdroidvcs-' + name
ftp.put(os.path.join('build/srclib', fv), fv)
send_dir(lib)
# Copy the metadata file too...
ftp.chdir('/home/vagrant/srclibs')
ftp.put(os.path.join('srclibs', name + '.txt'),
name + '.txt')
# Copy the main app source code
# (no need if it's a srclib)
if (not basesrclib) and os.path.exists(build_dir):
ftp.chdir('/home/vagrant/build')
fv = '.fdroidvcs-' + app['id']
ftp.put(os.path.join('build', fv), fv)
send_dir(build_dir)
# Execute the build script...
print "Starting build..."
chan = sshs.get_transport().open_session()
cmdline = 'python build.py --on-server'
if force:
cmdline += ' --force --test'
if options.verbose:
cmdline += ' --verbose'
cmdline += ' -p ' + app['id'] + ' --vercode ' + thisbuild['vercode']
chan.exec_command('bash -c ". ~/.bsenv && ' + cmdline + '"')
output = ''
error = ''
while not chan.exit_status_ready():
while chan.recv_ready():
output += chan.recv(1024)
while chan.recv_stderr_ready():
error += chan.recv_stderr(1024)
print "...getting exit status"
returncode = chan.recv_exit_status()
while True:
get = chan.recv(1024)
if len(get) == 0:
break
output += get
while True:
get = chan.recv_stderr(1024)
if len(get) == 0:
break
error += get
if returncode != 0:
raise BuildException("Build.py failed on server for %s:%s" % (app['id'], thisbuild['version']), output, error)
# Retrieve the built files...
print "Retrieving build output..."
if force:
ftp.chdir('/home/vagrant/tmp')
else:
ftp.chdir('/home/vagrant/unsigned')
apkfile = common.getapkname(app,thisbuild)
tarball = common.getsrcname(app,thisbuild)
try:
ftp.get(apkfile, os.path.join(output_dir, apkfile))
ftp.get(tarball, os.path.join(output_dir, tarball))
except:
raise BuildException("Build failed for %s:%s - missing output files" % (app['id'], thisbuild['version']), output, error)
ftp.close()
finally:
# Suspend the build server.
print "Suspending build server"
subprocess.call(['vagrant', 'suspend'], cwd='builder')
def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, install, force, onserver):
"""Do a build locally."""
# Prepare the source code...
root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild,
build_dir, srclib_dir, extlib_dir, onserver)
# We need to clean via the build tool in case the binary dirs are
# different from the default ones
p = None
if thisbuild.get('maven', 'no') != 'no':
print "Cleaning Maven project..."
cmd = [config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']]
if '@' in thisbuild['maven']:
maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@',1)[1])
maven_dir = os.path.normpath(maven_dir)
else:
maven_dir = root_dir
p = FDroidPopen(cmd, cwd=maven_dir)
elif thisbuild.get('gradle', 'no') != 'no':
print "Cleaning Gradle project..."
cmd = [config['gradle'], 'clean']
if '@' in thisbuild['gradle']:
gradle_dir = os.path.join(root_dir, thisbuild['gradle'].split('@',1)[1])
gradle_dir = os.path.normpath(gradle_dir)
else:
gradle_dir = root_dir
p = FDroidPopen(cmd, cwd=gradle_dir)
elif thisbuild.get('update', '.') != 'no' and thisbuild.get('kivy', 'no') == 'no':
print "Cleaning Ant project..."
cmd = ['ant', 'clean']
p = FDroidPopen(cmd, cwd=root_dir)
if p is not None and p.returncode != 0:
raise BuildException("Error cleaning %s:%s" %
(app['id'], thisbuild['version']), p.stdout, p.stderr)
# Also clean jni
print "Cleaning jni dirs..."
for baddir in [
'libs/armeabi-v7a', 'libs/armeabi',
'libs/mips', 'libs/x86', 'obj']:
badpath = os.path.join(build_dir, baddir)
if os.path.exists(badpath):
print "Removing '%s'" % badpath
shutil.rmtree(badpath)
# Scan before building...
print "Scanning source for common problems..."
buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
if len(buildprobs) > 0:
print 'Scanner found ' + str(len(buildprobs)) + ' problems:'
for problem in buildprobs:
print '...' + problem
if not force:
raise BuildException("Can't build due to " +
str(len(buildprobs)) + " scanned problems")
# Build the source tarball right before we build the release...
print "Creating source tarball..."
tarname = common.getsrcname(app,thisbuild)
tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz")
def tarexc(f):
for vcs_dir in ['.svn', '.git', '.hg', '.bzr']:
if f.endswith(vcs_dir):
return True
return False
tarball.add(build_dir, tarname, exclude=tarexc)
tarball.close()
# Run a build command if one is required...
if 'build' in thisbuild:
cmd = common.replace_config_vars(thisbuild['build'])
# Substitute source library paths into commands...
for name, number, libpath in srclibpaths:
libpath = os.path.relpath(libpath, root_dir)
cmd = cmd.replace('$$' + name + '$$', libpath)
if options.verbose:
print "Running 'build' commands in %s" % root_dir
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
raise BuildException("Error running build command for %s:%s" %
(app['id'], thisbuild['version']), p.stdout, p.stderr)
# Build native stuff if required...
if thisbuild.get('buildjni') not in (None, 'no'):
print "Building native libraries..."
jni_components = thisbuild.get('buildjni')
if jni_components == 'yes':
jni_components = ['']
else:
jni_components = [c.strip() for c in jni_components.split(';')]
ndkbuild = os.path.join(config['ndk_path'], "ndk-build")
for d in jni_components:
if options.verbose:
print "Running ndk-build in " + root_dir + '/' + d
manifest = root_dir + '/' + d + '/AndroidManifest.xml'
if os.path.exists(manifest):
# Read and write the whole AM.xml to fix newlines and avoid
# the ndk r8c or later 'wordlist' errors. The outcome of this
# under gnu/linux is the same as when using tools like
# dos2unix, but the native python way is faster and will
# work in non-unix systems.
manifest_text = open(manifest, 'U').read()
open(manifest, 'w').write(manifest_text)
# In case the AM.xml read was big, free the memory
del manifest_text
p = FDroidPopen([ndkbuild], cwd=os.path.join(root_dir,d))
if p.returncode != 0:
raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout, p.stderr)
p = None
# Build the release...
if thisbuild.get('maven', 'no') != 'no':
print "Building Maven project..."
if '@' in thisbuild['maven']:
maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@',1)[1])
else:
maven_dir = root_dir
mvncmd = [config['mvn3'], '-Dandroid.sdk.path=' + config['sdk_path']]
if install:
mvncmd += ['-Dandroid.sign.debug=true', 'package', 'android:deploy']
else:
mvncmd += ['-Dandroid.sign.debug=false', '-Dandroid.release=true', 'package']
if 'target' in thisbuild:
target = thisbuild["target"].split('-')[1]
subprocess.call(['sed', '-i',
's@<platform>[0-9]*</platform>@<platform>'+target+'</platform>@g',
'pom.xml'], cwd=root_dir)
if '@' in thisbuild['maven']:
subprocess.call(['sed', '-i',
's@<platform>[0-9]*</platform>@<platform>'+target+'</platform>@g',
'pom.xml'], cwd=maven_dir)
if 'mvnflags' in thisbuild:
mvncmd += thisbuild['mvnflags']
p = FDroidPopen(mvncmd, cwd=maven_dir)
bindir = os.path.join(root_dir, 'target')
elif thisbuild.get('kivy', 'no') != 'no':
print "Building Kivy project..."
spec = os.path.join(root_dir, 'buildozer.spec')
if not os.path.exists(spec):
raise BuildException("Expected to find buildozer-compatible spec at {0}"
.format(spec))
defaults = {'orientation': 'landscape', 'icon': '',
'permissions': '', 'android.api': "18"}
bconfig = ConfigParser(defaults, allow_no_value=True)
bconfig.read(spec)
distdir = 'python-for-android/dist/fdroid'
if os.path.exists(distdir):
shutil.rmtree(distdir)
modules = bconfig.get('app', 'requirements').split(',')
cmd = 'ANDROIDSDK=' + config['sdk_path']
cmd += ' ANDROIDNDK=' + config['ndk_path']
cmd += ' ANDROIDNDKVER=r9'
cmd += ' ANDROIDAPI=' + str(bconfig.get('app', 'android.api'))
cmd += ' VIRTUALENV=virtualenv'
cmd += ' ./distribute.sh'
cmd += ' -m ' + "'" + ' '.join(modules) + "'"
cmd += ' -d fdroid'
if subprocess.call(cmd, cwd='python-for-android', shell=True) != 0:
raise BuildException("Distribute build failed")
cid = bconfig.get('app', 'package.domain') + '.' + bconfig.get('app', 'package.name')
if cid != app['id']:
raise BuildException("Package ID mismatch between metadata and spec")
orientation = bconfig.get('app', 'orientation', 'landscape')
if orientation == 'all':
orientation = 'sensor'
cmd = ['./build.py'
'--dir', root_dir,
'--name', bconfig.get('app', 'title'),
'--package', app['id'],
'--version', bconfig.get('app', 'version'),
'--orientation', orientation,
]
perms = bconfig.get('app', 'permissions')
for perm in perms.split(','):
cmd.extend(['--permission', perm])
if config.get('app', 'fullscreen') == 0:
cmd.append('--window')
icon = bconfig.get('app', 'icon.filename')
if icon:
cmd.extend(['--icon', os.path.join(root_dir, icon)])
cmd.append('release')
p = FDroidPopen(cmd, cwd=distdir)
elif thisbuild.get('gradle', 'no') != 'no':
print "Building Gradle project..."
if '@' in thisbuild['gradle']:
flavour = thisbuild['gradle'].split('@')[0]
gradle_dir = thisbuild['gradle'].split('@')[1]
gradle_dir = os.path.join(root_dir, gradle_dir)
else:
flavour = thisbuild['gradle']
gradle_dir = root_dir
if 'compilesdk' in thisbuild:
level = thisbuild["compilesdk"].split('-')[1]
subprocess.call(['sed', '-i',
's@compileSdkVersion[ ]*[0-9]*@compileSdkVersion '+level+'@g',
'build.gradle'], cwd=root_dir)
if '@' in thisbuild['gradle']:
subprocess.call(['sed', '-i',
's@compileSdkVersion[ ]*[0-9]*@compileSdkVersion '+level+'@g',
'build.gradle'], cwd=gradle_dir)
for root, dirs, files in os.walk(build_dir):
for f in files:
if f == 'build.gradle':
adapt_gradle(os.path.join(root, f))
break
if flavour in ['main', 'yes', '']:
flavour = ''
commands = [config['gradle']]
if 'preassemble' in thisbuild:
for task in thisbuild['preassemble'].split():
commands.append(task)
if install:
commands += ['assemble'+flavour+'Debug', 'install'+flavour+'Debug']
else:
commands += ['assemble'+flavour+'Release']
p = FDroidPopen(commands, cwd=gradle_dir)
else:
print "Building Ant project..."
cmd = ['ant']
if install:
cmd += ['debug','install']
elif 'antcommand' in thisbuild:
cmd += [thisbuild['antcommand']]
else:
cmd += ['release']
p = FDroidPopen(cmd, cwd=root_dir)
bindir = os.path.join(root_dir, 'bin')
if p.returncode != 0:
raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout, p.stderr)
print "Successfully built version " + thisbuild['version'] + ' of ' + app['id']
if install:
return
# Find the apk name in the output...
if 'bindir' in thisbuild:
bindir = os.path.join(build_dir, thisbuild['bindir'])
if thisbuild.get('maven', 'no') != 'no':
stdout_apk = '\n'.join([
line for line in p.stdout.splitlines() if any(a in line for a in ('.apk','.ap_'))])
m = re.match(r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk",
stdout_apk, re.S|re.M)
if not m:
m = re.match(r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]",
stdout_apk, re.S|re.M)
if not m:
m = re.match(r'.*^\[INFO\] [^$]*aapt \[package,[^$]*' + bindir + r'/([^/]+)\.ap[_k][,\]]',
stdout_apk, re.S|re.M)
if not m:
raise BuildException('Failed to find output')
src = m.group(1)
src = os.path.join(bindir, src) + '.apk'
elif thisbuild.get('kivy', 'no') != 'no':
src = 'python-for-android/dist/default/bin/{0}-{1}-release.apk'.format(
bconfig.get('app', 'title'), bconfig.get('app', 'version'))
elif thisbuild.get('gradle', 'no') != 'no':
dd = build_dir
if 'subdir' in thisbuild:
dd = os.path.join(dd, thisbuild['subdir'])
if flavour in ['main', 'yes', '']:
name = '-'.join([os.path.basename(dd), 'release', 'unsigned'])
else:
name = '-'.join([os.path.basename(dd), flavour, 'release', 'unsigned'])
src = os.path.join(dd, 'build', 'apk', name+'.apk')
else:
stdout_apk = '\n'.join([
line for line in p.stdout.splitlines() if '.apk' in line])
src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk,
re.S|re.M).group(1)
src = os.path.join(bindir, src)
# Make sure it's not debuggable...
if common.isApkDebuggable(src, config):
raise BuildException("APK is debuggable")
# By way of a sanity check, make sure the version and version
# code in our new apk match what we expect...
print "Checking " + src
if not os.path.exists(src):
raise BuildException("Unsigned apk is not at expected location of " + src)
p = subprocess.Popen([os.path.join(config['sdk_path'],
'build-tools', config['build_tools'], 'aapt'),
'dump', 'badging', src],
stdout=subprocess.PIPE)
output = p.communicate()[0]
vercode = None
version = None
foundid = None
for line in output.splitlines():
if line.startswith("package:"):
pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
foundid = re.match(pat, line).group(1)
pat = re.compile(".*versionCode='([0-9]*)'.*")
vercode = re.match(pat, line).group(1)
pat = re.compile(".*versionName='([^']*)'.*")
version = re.match(pat, line).group(1)
if thisbuild['novcheck']:
vercode = thisbuild['vercode']
version = thisbuild['version']
if not version or not vercode:
raise BuildException("Could not find version information in build in output")
if not foundid:
raise BuildException("Could not find package ID in output")
if foundid != app['id']:
raise BuildException("Wrong package ID - build " + foundid + " but expected " + app['id'])
# Some apps (e.g. Timeriffic) have had the bonkers idea of
# including the entire changelog in the version number. Remove
# it so we can compare. (TODO: might be better to remove it
# before we compile, in fact)
index = version.find(" //")
if index != -1:
version = version[:index]
if (version != thisbuild['version'] or
vercode != thisbuild['vercode']):
raise BuildException(("Unexpected version/version code in output;"
" APK: '%s' / '%s', "
" Expected: '%s' / '%s'")
% (version, str(vercode), thisbuild['version'], str(thisbuild['vercode']))
)
# Copy the unsigned apk to our destination directory for further
# processing (by publish.py)...
dest = os.path.join(output_dir, common.getapkname(app,thisbuild))
shutil.copyfile(src, dest)
# Move the source tarball into the output directory...
if output_dir != tmp_dir:
shutil.move(os.path.join(tmp_dir, tarname),
os.path.join(output_dir, tarname))