elif line.startswith("uses-feature:"):
perm = re.match(string_pat, line).group(1)
#Filter out this, it's only added with the latest SDK tools and
#causes problems for lots of apps.
if (perm != "android.hardware.screen.portrait" and
perm != "android.hardware.screen.landscape"):
if perm.startswith("android.feature."):
perm = perm[16:]
thisinfo['features'].append(perm)
if not 'sdkversion' in thisinfo:
print " WARNING: no SDK version information found"
thisinfo['sdkversion'] = 0
# Check for debuggable apks...
if common.isApkDebuggable(apkfile, config):
print "WARNING: {0} is debuggable... {1}".format(apkfile, line)
# Calculate the sha256...
sha = hashlib.sha256()
with open(apkfile, 'rb') as f:
while True:
t = f.read(1024)
if len(t) == 0:
break
sha.update(t)
thisinfo['sha256'] = sha.hexdigest()
# Get the signature (or md5 of, to be precise)...
getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
if not os.path.exists(getsig_dir + "/getsig.class"):
def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir,
extlib_dir, tmp_dir, install, force, verbose, onserver):
"""Do a build locally."""
# Prepare the source code...
root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild,
build_dir, srclib_dir,
extlib_dir, sdk_path,
ndk_path, javacc_path, mvn3,
verbose, onserver)
# Scan before building...
buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
if len(buildprobs) > 0:
print 'Scanner found ' + str(len(buildprobs)) + ' problems:'
for problem in buildprobs:
print '...' + problem
if not force:
raise BuildException("Can't build due to " + str(len(buildprobs)) +
" scanned problems")
# Build the source tarball right before we build the release...
tarname = app['id'] + '_' + thisbuild['vercode'] + '_src'
tarball = tarfile.open(os.path.join(tmp_dir, tarname + '.tar.gz'), "w:gz")
def tarexc(f):
for vcs_dir in ['.svn', '.git', '.hg', '.bzr']:
if f.endswith(vcs_dir):
return True
return False
tarball.add(build_dir, tarname, exclude=tarexc)
tarball.close()
# Run a build command if one is required...
if 'build' in thisbuild:
prebuild = thisbuild['build']
# Substitute source library paths into prebuild commands...
for name, libpath in srclibpaths:
libpath = os.path.relpath(libpath, root_dir)
prebuild = prebuild.replace('$$' + name + '$$', libpath)
prebuild = prebuild.replace('$$SDK$$', sdk_path)
prebuild = prebuild.replace('$$NDK$$', ndk_path)
prebuild = prebuild.replace('$$MVN3$$', mvn3)
p = subprocess.Popen(prebuild,
cwd=root_dir,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = p.communicate()
if p.returncode != 0:
raise BuildException("Error running build command", out, err)
# Build native stuff if required...
if thisbuild.get('buildjni') not in (None, 'no'):
jni_components = thisbuild.get('buildjni')
if jni_components == 'yes':
jni_components = ['']
else:
jni_components = jni_components.split(';')
ndkbuild = os.path.join(ndk_path, "ndk-build")
for d in jni_components:
if options.verbose:
print "Running ndk-build in " + root_dir + '/' + d
manifest = root_dir + '/' + d + '/AndroidManifest.xml'
if os.path.exists(manifest):
# Read and write the whole AM.xml to fix newlines and avoid
# the ndk r8c or later 'wordlist' errors. The outcome of this
# under gnu/linux is the same as when using tools like
# dos2unix, but the native python way is faster and will
# work in non-unix systems.
manifest_text = open(manifest, 'U').read()
open(manifest, 'w').write(manifest_text)
# In case the AM.xml read was big, free the memory
del manifest_text
p = subprocess.Popen([ndkbuild],
cwd=root_dir + '/' + d,
stdout=subprocess.PIPE)
output = p.communicate()[0]
if p.returncode != 0:
print output
raise BuildException("NDK build failed for %s:%s" %
(app['id'], thisbuild['version']))
# Build the release...
if 'maven' in thisbuild:
mvncmd = [mvn3, 'clean', 'package', '-Dandroid.sdk.path=' + sdk_path]
if install:
mvncmd += ['-Dandroid.sign.debug=true']
else:
mvncmd += ['-Dandroid.sign.debug=false', '-Dandroid.release=true']
if 'mvnflags' in thisbuild:
mvncmd += thisbuild['mvnflags']
p = subprocess.Popen(mvncmd,
cwd=root_dir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
else:
if install:
antcommands = ['debug', 'install']
elif 'antcommand' in thisbuild:
antcommands = [thisbuild['antcommand']]
else:
antcommands = ['release']
p = subprocess.Popen(['ant'] + antcommands,
cwd=root_dir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, error = p.communicate()
if p.returncode != 0:
raise BuildException(
"Build failed for %s:%s" % (app['id'], thisbuild['version']),
output.strip(), error.strip())
if verbose:
print output
if install:
if 'maven' in thisbuild:
p = subprocess.Popen(
[mvn3, 'android:deploy', '-Dandroid.sdk.path=' + sdk_path],
cwd=root_dir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, error = p.communicate()
if p.returncode != 0:
raise BuildException(
"Warning: Could not deploy %s:%s" %
(app['id'], thisbuild['version']), output.strip(),
error.strip())
return
print "Build successful"
# Find the apk name in the output...
if 'bindir' in thisbuild:
bindir = os.path.join(build_dir, thisbuild['bindir'])
else:
bindir = os.path.join(root_dir, 'bin')
if thisbuild.get('initfun', 'no') == "yes":
# Special case (again!) for funambol...
src = ("funambol-android-sync-client-" + thisbuild['version'] +
"-unsigned.apk")
src = os.path.join(bindir, src)
elif 'maven' in thisbuild:
m = re.match(r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", output,
re.S | re.M)
if not m:
m = re.match(
r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk",
output, re.S | re.M)
if not m:
# This format is found in com.github.mobile, com.yubico.yubitotp and com.botbrew.basil for example...
m = re.match(
r".*^\[INFO\] [^$]*aapt \[package,[^$]*" + app['id'] + "/" +
thisbuild['bindir'] + "/([^/]+)\.ap[_k][,\]]", output,
re.S | re.M)
if not m:
print output
raise BuildException('Failed to find output')
src = m.group(1)
src = os.path.join(bindir, src) + '.apk'
else:
src = re.match(r".*^.*Creating (.+) for release.*$.*", output,
re.S | re.M).group(1)
src = os.path.join(bindir, src)
# Make sure it's not debuggable...
if not install and common.isApkDebuggable(src, sdk_path):
raise BuildException("APK is debuggable")
# By way of a sanity check, make sure the version and version
# code in our new apk match what we expect...
print "Checking " + src
if not os.path.exists(src):
raise BuildException("Unsigned apk is not at expected location of " +
src)
p = subprocess.Popen([
os.path.join(sdk_path, 'platform-tools', 'aapt'), 'dump', 'badging',
src
],
stdout=subprocess.PIPE)
output = p.communicate()[0]
if thisbuild.get('novcheck', 'no') == "yes":
vercode = thisbuild['vercode']
version = thisbuild['version']
else:
vercode = None
version = None
foundid = None
for line in output.splitlines():
if line.startswith("package:"):
pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
foundid = re.match(pat, line).group(1)
pat = re.compile(".*versionCode='([0-9]*)'.*")
vercode = re.match(pat, line).group(1)
pat = re.compile(".*versionName='([^']*)'.*")
version = re.match(pat, line).group(1)
if not version or not vercode:
raise BuildException(
"Could not find version information in build in output")
if not foundid:
raise BuildException("Could not find package ID in output")
if foundid != app['id']:
raise BuildException("Wrong package ID - build " + foundid +
" but expected " + app['id'])
# Some apps (e.g. Timeriffic) have had the bonkers idea of
# including the entire changelog in the version number. Remove
# it so we can compare. (TODO: might be better to remove it
# before we compile, in fact)
index = version.find(" //")
if index != -1:
version = version[:index]
if (version != thisbuild['version'] or vercode != thisbuild['vercode']):
raise BuildException(("Unexpected version/version code in output;"
" APK: '%s' / '%s', "
" Expected: '%s' / '%s'") %
(version, str(vercode), thisbuild['version'],
str(thisbuild['vercode'])))
# Copy the unsigned apk to our destination directory for further
# processing (by publish.py)...
dest = os.path.join(output_dir,
app['id'] + '_' + thisbuild['vercode'] + '.apk')
shutil.copyfile(src, dest)
# Move the source tarball into the output directory...
if output_dir != tmp_dir:
tarfilename = tarname + '.tar.gz'
shutil.move(os.path.join(tmp_dir, tarfilename),
os.path.join(output_dir, tarfilename))
def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir,
extlib_dir, tmp_dir, force, onserver):
"""Do a build locally."""
if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
if not config['ndk_path']:
logging.critical("$ANDROID_NDK is not set!")
sys.exit(3)
elif not os.path.isdir(config['sdk_path']):
logging.critical(
"$ANDROID_NDK points to a non-existing directory!")
sys.exit(3)
# Prepare the source code...
root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild,
build_dir, srclib_dir,
extlib_dir, onserver)
# We need to clean via the build tool in case the binary dirs are
# different from the default ones
p = None
if thisbuild['type'] == 'maven':
logging.info("Cleaning Maven project...")
cmd = [
config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']
]
if '@' in thisbuild['maven']:
maven_dir = os.path.join(root_dir,
thisbuild['maven'].split('@', 1)[1])
maven_dir = os.path.normpath(maven_dir)
else:
maven_dir = root_dir
p = FDroidPopen(cmd, cwd=maven_dir)
elif thisbuild['type'] == 'gradle':
logging.info("Cleaning Gradle project...")
cmd = [config['gradle'], 'clean']
adapt_gradle(build_dir)
for name, number, libpath in srclibpaths:
adapt_gradle(libpath)
p = FDroidPopen(cmd, cwd=root_dir)
elif thisbuild['type'] == 'kivy':
pass
elif thisbuild['type'] == 'ant':
logging.info("Cleaning Ant project...")
p = FDroidPopen(['ant', 'clean'], cwd=root_dir)
if p is not None and p.returncode != 0:
raise BuildException(
"Error cleaning %s:%s" % (app['id'], thisbuild['version']),
p.output)
for root, dirs, files in os.walk(build_dir):
# Don't remove possibly necessary 'gradle' dirs if 'gradlew' is not there
if 'gradlew' in files:
logging.debug("Getting rid of Gradle wrapper stuff in %s" % root)
os.remove(os.path.join(root, 'gradlew'))
if 'gradlew.bat' in files:
os.remove(os.path.join(root, 'gradlew.bat'))
if 'gradle' in dirs:
shutil.rmtree(os.path.join(root, 'gradle'))
if not options.skipscan:
# Scan before building...
logging.info("Scanning source for common problems...")
count = common.scan_source(build_dir, root_dir, thisbuild)
if count > 0:
if force:
logging.warn('Scanner found %d problems:' % count)
else:
raise BuildException(
"Can't build due to %d errors while scanning" % count)
if not options.notarball:
# Build the source tarball right before we build the release...
logging.info("Creating source tarball...")
tarname = common.getsrcname(app, thisbuild)
tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz")
def tarexc(f):
return any(f.endswith(s) for s in ['.svn', '.git', '.hg', '.bzr'])
tarball.add(build_dir, tarname, exclude=tarexc)
tarball.close()
if onserver:
manifest = os.path.join(root_dir, 'AndroidManifest.xml')
if os.path.exists(manifest):
homedir = os.path.expanduser('~')
with open(os.path.join(homedir, 'buildserverid'), 'r') as f:
buildserverid = f.read()
with open(os.path.join(homedir, 'fdroidserverid'), 'r') as f:
fdroidserverid = f.read()
with open(manifest, 'r') as f:
manifestcontent = f.read()
manifestcontent = manifestcontent.replace(
'</manifest>',
'<fdroid buildserverid="' + buildserverid + '"' +
' fdroidserverid="' + fdroidserverid + '"' + '/></manifest>')
with open(manifest, 'w') as f:
f.write(manifestcontent)
# Run a build command if one is required...
if thisbuild['build']:
logging.info("Running 'build' commands in %s" % root_dir)
cmd = common.replace_config_vars(thisbuild['build'])
# Substitute source library paths into commands...
for name, number, libpath in srclibpaths:
libpath = os.path.relpath(libpath, root_dir)
cmd = cmd.replace('$$' + name + '$$', libpath)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
raise BuildException(
"Error running build command for %s:%s" %
(app['id'], thisbuild['version']), p.output)
# Build native stuff if required...
if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
logging.info("Building the native code")
jni_components = thisbuild['buildjni']
if jni_components == ['yes']:
jni_components = ['']
cmd = [os.path.join(config['ndk_path'], "ndk-build"), "-j1"]
for d in jni_components:
if d:
logging.info("Building native code in '%s'" % d)
else:
logging.info("Building native code in the main project")
manifest = root_dir + '/' + d + '/AndroidManifest.xml'
if os.path.exists(manifest):
# Read and write the whole AM.xml to fix newlines and avoid
# the ndk r8c or later 'wordlist' errors. The outcome of this
# under gnu/linux is the same as when using tools like
# dos2unix, but the native python way is faster and will
# work in non-unix systems.
manifest_text = open(manifest, 'U').read()
open(manifest, 'w').write(manifest_text)
# In case the AM.xml read was big, free the memory
del manifest_text
p = FDroidPopen(cmd, cwd=os.path.join(root_dir, d))
if p.returncode != 0:
raise BuildException(
"NDK build failed for %s:%s" %
(app['id'], thisbuild['version']), p.output)
p = None
# Build the release...
if thisbuild['type'] == 'maven':
logging.info("Building Maven project...")
if '@' in thisbuild['maven']:
maven_dir = os.path.join(root_dir,
thisbuild['maven'].split('@', 1)[1])
else:
maven_dir = root_dir
mvncmd = [
config['mvn3'], '-Dandroid.sdk.path=' + config['sdk_path'],
'-Dmaven.jar.sign.skip=true', '-Dmaven.test.skip=true',
'-Dandroid.sign.debug=false', '-Dandroid.release=true', 'package'
]
if thisbuild['target']:
target = thisbuild["target"].split('-')[1]
FDroidPopen([
'sed', '-i', 's@<platform>[0-9]*</platform>@<platform>' +
target + '</platform>@g', 'pom.xml'
],
cwd=root_dir)
if '@' in thisbuild['maven']:
FDroidPopen([
'sed', '-i', 's@<platform>[0-9]*</platform>@<platform>' +
target + '</platform>@g', 'pom.xml'
],
cwd=maven_dir)
p = FDroidPopen(mvncmd, cwd=maven_dir)
bindir = os.path.join(root_dir, 'target')
elif thisbuild['type'] == 'kivy':
logging.info("Building Kivy project...")
spec = os.path.join(root_dir, 'buildozer.spec')
if not os.path.exists(spec):
raise BuildException(
"Expected to find buildozer-compatible spec at {0}".format(
spec))
defaults = {
'orientation': 'landscape',
'icon': '',
'permissions': '',
'android.api': "18"
}
bconfig = ConfigParser(defaults, allow_no_value=True)
bconfig.read(spec)
distdir = 'python-for-android/dist/fdroid'
if os.path.exists(distdir):
shutil.rmtree(distdir)
modules = bconfig.get('app', 'requirements').split(',')
cmd = 'ANDROIDSDK=' + config['sdk_path']
cmd += ' ANDROIDNDK=' + config['ndk_path']
cmd += ' ANDROIDNDKVER=r9'
cmd += ' ANDROIDAPI=' + str(bconfig.get('app', 'android.api'))
cmd += ' VIRTUALENV=virtualenv'
cmd += ' ./distribute.sh'
cmd += ' -m ' + "'" + ' '.join(modules) + "'"
cmd += ' -d fdroid'
p = FDroidPopen(cmd, cwd='python-for-android', shell=True)
if p.returncode != 0:
raise BuildException("Distribute build failed")
cid = bconfig.get('app', 'package.domain') + '.' + bconfig.get(
'app', 'package.name')
if cid != app['id']:
raise BuildException(
"Package ID mismatch between metadata and spec")
orientation = bconfig.get('app', 'orientation', 'landscape')
if orientation == 'all':
orientation = 'sensor'
cmd = [
'./build.py'
'--dir', root_dir, '--name',
bconfig.get('app', 'title'), '--package', app['id'], '--version',
bconfig.get('app', 'version'), '--orientation', orientation
]
perms = bconfig.get('app', 'permissions')
for perm in perms.split(','):
cmd.extend(['--permission', perm])
if config.get('app', 'fullscreen') == 0:
cmd.append('--window')
icon = bconfig.get('app', 'icon.filename')
if icon:
cmd.extend(['--icon', os.path.join(root_dir, icon)])
cmd.append('release')
p = FDroidPopen(cmd, cwd=distdir)
elif thisbuild['type'] == 'gradle':
logging.info("Building Gradle project...")
flavours = thisbuild['gradle'].split(',')
if len(flavours) == 1 and flavours[0] in ['main', 'yes', '']:
flavours[0] = ''
commands = [config['gradle']]
if thisbuild['preassemble']:
commands += thisbuild['preassemble'].split()
flavours_cmd = ''.join(flavours)
if flavours_cmd:
flavours_cmd = flavours_cmd[0].upper() + flavours_cmd[1:]
commands += ['assemble' + flavours_cmd + 'Release']
# Avoid having to use lintOptions.abortOnError false
if thisbuild['gradlepluginver'] >= LooseVersion('0.7'):
with open(os.path.join(root_dir, 'build.gradle'), "a") as f:
f.write(
"\nandroid { lintOptions { checkReleaseBuilds false } }\n")
p = FDroidPopen(commands, cwd=root_dir)
elif thisbuild['type'] == 'ant':
logging.info("Building Ant project...")
cmd = ['ant']
if thisbuild['antcommand']:
cmd += [thisbuild['antcommand']]
else:
cmd += ['release']
p = FDroidPopen(cmd, cwd=root_dir)
bindir = os.path.join(root_dir, 'bin')
if p is not None and p.returncode != 0:
raise BuildException(
"Build failed for %s:%s" % (app['id'], thisbuild['version']),
p.output)
logging.info("Successfully built version " + thisbuild['version'] +
' of ' + app['id'])
if thisbuild['type'] == 'maven':
stdout_apk = '\n'.join([
line for line in p.output.splitlines()
if any(a in line for a in ('.apk', '.ap_', '.jar'))
])
m = re.match(r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", stdout_apk,
re.S | re.M)
if not m:
m = re.match(
r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]",
stdout_apk, re.S | re.M)
if not m:
m = re.match(
r'.*^\[INFO\] [^$]*aapt \[package,[^$]*' + bindir +
r'/([^/]+)\.ap[_k][,\]]', stdout_apk, re.S | re.M)
if not m:
m = re.match(
r".*^\[INFO\] Building jar: .*/" + bindir + r"/(.+)\.jar",
stdout_apk, re.S | re.M)
if not m:
raise BuildException('Failed to find output')
src = m.group(1)
src = os.path.join(bindir, src) + '.apk'
elif thisbuild['type'] == 'kivy':
src = 'python-for-android/dist/default/bin/{0}-{1}-release.apk'.format(
bconfig.get('app', 'title'), bconfig.get('app', 'version'))
elif thisbuild['type'] == 'gradle':
if thisbuild['gradlepluginver'] >= LooseVersion('0.11'):
apks_dir = os.path.join(root_dir, 'build', 'outputs', 'apk')
else:
apks_dir = os.path.join(root_dir, 'build', 'apk')
apks = glob.glob(os.path.join(apks_dir, '*-release-unsigned.apk'))
if len(apks) > 1:
raise BuildException(
'More than one resulting apks found in %s' % apks_dir,
'\n'.join(apks))
if len(apks) < 1:
raise BuildException('Failed to find gradle output in %s' %
apks_dir)
src = apks[0]
elif thisbuild['type'] == 'ant':
stdout_apk = '\n'.join(
[line for line in p.output.splitlines() if '.apk' in line])
src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk,
re.S | re.M).group(1)
src = os.path.join(bindir, src)
elif thisbuild['type'] == 'raw':
src = os.path.join(root_dir, thisbuild['output'])
src = os.path.normpath(src)
# Make sure it's not debuggable...
if common.isApkDebuggable(src, config):
raise BuildException("APK is debuggable")
# By way of a sanity check, make sure the version and version
# code in our new apk match what we expect...
logging.debug("Checking " + src)
if not os.path.exists(src):
raise BuildException("Unsigned apk is not at expected location of " +
src)
p = SilentPopen([config['aapt'], 'dump', 'badging', src])
vercode = None
version = None
foundid = None
nativecode = None
for line in p.output.splitlines():
if line.startswith("package:"):
pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
m = pat.match(line)
if m:
foundid = m.group(1)
pat = re.compile(".*versionCode='([0-9]*)'.*")
m = pat.match(line)
if m:
vercode = m.group(1)
pat = re.compile(".*versionName='([^']*)'.*")
m = pat.match(line)
if m:
version = m.group(1)
elif line.startswith("native-code:"):
nativecode = line[12:]
# Ignore empty strings or any kind of space/newline chars that we don't
# care about
if nativecode is not None:
nativecode = nativecode.strip()
nativecode = None if not nativecode else nativecode
if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
if nativecode is None:
raise BuildException(
"Native code should have been built but none was packaged")
if thisbuild['novcheck']:
vercode = thisbuild['vercode']
version = thisbuild['version']
if not version or not vercode:
raise BuildException(
"Could not find version information in build in output")
if not foundid:
raise BuildException("Could not find package ID in output")
if foundid != app['id']:
raise BuildException("Wrong package ID - build " + foundid +
" but expected " + app['id'])
# Some apps (e.g. Timeriffic) have had the bonkers idea of
# including the entire changelog in the version number. Remove
# it so we can compare. (TODO: might be better to remove it
# before we compile, in fact)
index = version.find(" //")
if index != -1:
version = version[:index]
if (version != thisbuild['version'] or vercode != thisbuild['vercode']):
raise BuildException(("Unexpected version/version code in output;"
" APK: '%s' / '%s', "
" Expected: '%s' / '%s'") %
(version, str(vercode), thisbuild['version'],
str(thisbuild['vercode'])))
# Copy the unsigned apk to our destination directory for further
# processing (by publish.py)...
dest = os.path.join(output_dir, common.getapkname(app, thisbuild))
shutil.copyfile(src, dest)
# Move the source tarball into the output directory...
if output_dir != tmp_dir and not options.notarball:
shutil.move(os.path.join(tmp_dir, tarname),
os.path.join(output_dir, tarname))
elif line.startswith("uses-feature:"):
perm = re.match(string_pat, line).group(1)
#Filter out this, it's only added with the latest SDK tools and
#causes problems for lots of apps.
if (perm != "android.hardware.screen.portrait" and
perm != "android.hardware.screen.landscape"):
if perm.startswith("android.feature."):
perm = perm[16:]
thisinfo['features'].append(perm)
if not 'sdkversion' in thisinfo:
print " WARNING: no SDK version information found"
thisinfo['sdkversion'] = 0
# Check for debuggable apks...
if common.isApkDebuggable(apkfile, sdk_path):
print "WARNING: {0} is debuggable... {1}".format(apkfile, line)
# Calculate the md5 and sha256...
m = hashlib.md5()
sha = hashlib.sha256()
with open(apkfile, 'rb') as f:
while True:
t = f.read(1024)
if len(t) == 0:
break
m.update(t)
sha.update(t)
thisinfo['md5'] = m.hexdigest()
thisinfo['sha256'] = sha.hexdigest()
def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, install, force, verbose, onserver):
"""Do a build locally."""
# Prepare the source code...
root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild,
build_dir, srclib_dir, extlib_dir, sdk_path, ndk_path,
javacc_path, mvn3, verbose, onserver)
# Scan before building...
buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
if len(buildprobs) > 0:
print 'Scanner found ' + str(len(buildprobs)) + ' problems:'
for problem in buildprobs:
print '...' + problem
if not force:
raise BuildException("Can't build due to " +
str(len(buildprobs)) + " scanned problems")
# Build the source tarball right before we build the release...
tarname = app['id'] + '_' + thisbuild['vercode'] + '_src'
tarball = tarfile.open(os.path.join(tmp_dir,
tarname + '.tar.gz'), "w:gz")
def tarexc(f):
for vcs_dir in ['.svn', '.git', '.hg', '.bzr']:
if f.endswith(vcs_dir):
return True
return False
tarball.add(build_dir, tarname, exclude=tarexc)
tarball.close()
# Run a build command if one is required...
if 'build' in thisbuild:
prebuild = thisbuild['build']
# Substitute source library paths into prebuild commands...
for name, libpath in srclibpaths:
libpath = os.path.relpath(libpath, root_dir)
prebuild = prebuild.replace('$$' + name + '$$', libpath)
prebuild = prebuild.replace('$$SDK$$', sdk_path)
prebuild = prebuild.replace('$$NDK$$', ndk_path)
prebuild = prebuild.replace('$$MVN3$$', mvn3)
p = subprocess.Popen(prebuild, cwd=root_dir, shell=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
if p.returncode != 0:
raise BuildException("Error running build command", out, err)
# Build native stuff if required...
if thisbuild.get('buildjni') not in (None, 'no'):
jni_components = thisbuild.get('buildjni')
if jni_components == 'yes':
jni_components = ['']
else:
jni_components = jni_components.split(';')
ndkbuild = os.path.join(ndk_path, "ndk-build")
for d in jni_components:
if options.verbose:
print "Running ndk-build in " + root_dir + '/' + d
manifest = root_dir + '/' + d + '/AndroidManifest.xml'
if os.path.exists(manifest):
# Read and write the whole AM.xml to fix newlines and avoid
# the ndk r8c or later 'wordlist' errors. The outcome of this
# under gnu/linux is the same as when using tools like
# dos2unix, but the native python way is faster and will
# work in non-unix systems.
manifest_text = open(manifest, 'U').read()
open(manifest, 'w').write(manifest_text)
# In case the AM.xml read was big, free the memory
del manifest_text
p = subprocess.Popen([ndkbuild], cwd=root_dir + '/' + d,
stdout=subprocess.PIPE)
output = p.communicate()[0]
if p.returncode != 0:
print output
raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version']))
# Build the release...
if 'maven' in thisbuild:
mvncmd = [mvn3, 'clean', 'package', '-Dandroid.sdk.path=' + sdk_path]
if install:
mvncmd += ['-Dandroid.sign.debug=true']
else:
mvncmd += ['-Dandroid.sign.debug=false', '-Dandroid.release=true']
if 'mvnflags' in thisbuild:
mvncmd += thisbuild['mvnflags']
p = subprocess.Popen(mvncmd, cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
else:
if install:
antcommands = ['debug','install']
elif 'antcommand' in thisbuild:
antcommands = [thisbuild['antcommand']]
else:
antcommands = ['release']
p = subprocess.Popen(['ant'] + antcommands, cwd=root_dir,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, error = p.communicate()
if p.returncode != 0:
raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip())
if verbose:
print output
if install:
if 'maven' in thisbuild:
p = subprocess.Popen([mvn3, 'android:deploy', '-Dandroid.sdk.path=' + sdk_path],
cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, error = p.communicate()
if p.returncode != 0:
raise BuildException("Warning: Could not deploy %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip())
return
print "Build successful"
# Find the apk name in the output...
if 'bindir' in thisbuild:
bindir = os.path.join(build_dir, thisbuild['bindir'])
else:
bindir = os.path.join(root_dir, 'bin')
if thisbuild.get('initfun', 'no') == "yes":
# Special case (again!) for funambol...
src = ("funambol-android-sync-client-" +
thisbuild['version'] + "-unsigned.apk")
src = os.path.join(bindir, src)
elif 'maven' in thisbuild:
m = re.match(r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk",
output, re.S|re.M)
if not m:
m = re.match(r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk",
output, re.S|re.M)
if not m:
# This format is found in com.github.mobile, com.yubico.yubitotp and com.botbrew.basil for example...
m = re.match(r".*^\[INFO\] [^$]*aapt \[package,[^$]*" + app['id'] + "/" + thisbuild['bindir'] + "/([^/]+)\.ap[_k][,\]]",
output, re.S|re.M)
if not m:
print output
raise BuildException('Failed to find output')
src = m.group(1)
src = os.path.join(bindir, src) + '.apk'
else:
src = re.match(r".*^.*Creating (.+) for release.*$.*", output,
re.S|re.M).group(1)
src = os.path.join(bindir, src)
# Make sure it's not debuggable...
if not install and common.isApkDebuggable(src, sdk_path):
raise BuildException("APK is debuggable")
# By way of a sanity check, make sure the version and version
# code in our new apk match what we expect...
print "Checking " + src
if not os.path.exists(src):
raise BuildException("Unsigned apk is not at expected location of " + src)
p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools',
'aapt'),
'dump', 'badging', src],
stdout=subprocess.PIPE)
output = p.communicate()[0]
if thisbuild.get('novcheck', 'no') == "yes":
vercode = thisbuild['vercode']
version = thisbuild['version']
else:
vercode = None
version = None
foundid = None
for line in output.splitlines():
if line.startswith("package:"):
pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
foundid = re.match(pat, line).group(1)
pat = re.compile(".*versionCode='([0-9]*)'.*")
vercode = re.match(pat, line).group(1)
pat = re.compile(".*versionName='([^']*)'.*")
version = re.match(pat, line).group(1)
if not version or not vercode:
raise BuildException("Could not find version information in build in output")
if not foundid:
raise BuildException("Could not find package ID in output")
if foundid != app['id']:
raise BuildException("Wrong package ID - build " + foundid + " but expected " + app['id'])
# Some apps (e.g. Timeriffic) have had the bonkers idea of
# including the entire changelog in the version number. Remove
# it so we can compare. (TODO: might be better to remove it
# before we compile, in fact)
index = version.find(" //")
if index != -1:
version = version[:index]
if (version != thisbuild['version'] or
vercode != thisbuild['vercode']):
raise BuildException(("Unexpected version/version code in output;"
" APK: '%s' / '%s', "
" Expected: '%s' / '%s'")
% (version, str(vercode), thisbuild['version'], str(thisbuild['vercode']))
)
# Copy the unsigned apk to our destination directory for further
# processing (by publish.py)...
dest = os.path.join(output_dir, app['id'] + '_' +
thisbuild['vercode'] + '.apk')
shutil.copyfile(src, dest)
# Move the source tarball into the output directory...
if output_dir != tmp_dir:
tarfilename = tarname + '.tar.gz'
shutil.move(os.path.join(tmp_dir, tarfilename),
os.path.join(output_dir, tarfilename))
def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh):
"""Do a build locally."""
ndk_path = build.ndk_path()
if build.buildjni and build.buildjni != ['no']:
if not ndk_path:
logging.critical("Android NDK version '%s' could not be found!" % build.ndk or 'r10e')
logging.critical("Configured versions:")
for k, v in config['ndk_paths'].iteritems():
if k.endswith("_orig"):
continue
logging.critical(" %s: %s" % (k, v))
sys.exit(3)
elif not os.path.isdir(ndk_path):
logging.critical("Android NDK '%s' is not a directory!" % ndk_path)
sys.exit(3)
# Set up environment vars that depend on each build
for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
common.env[n] = ndk_path
common.reset_env_path()
# Set up the current NDK to the PATH
common.add_to_env_path(ndk_path)
# Prepare the source code...
root_dir, srclibpaths = common.prepare_source(vcs, app, build,
build_dir, srclib_dir,
extlib_dir, onserver, refresh)
# We need to clean via the build tool in case the binary dirs are
# different from the default ones
p = None
gradletasks = []
bmethod = build.build_method()
if bmethod == 'maven':
logging.info("Cleaning Maven project...")
cmd = [config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']]
if '@' in build.maven:
maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1])
maven_dir = os.path.normpath(maven_dir)
else:
maven_dir = root_dir
p = FDroidPopen(cmd, cwd=maven_dir)
elif bmethod == 'gradle':
logging.info("Cleaning Gradle project...")
if build.preassemble:
gradletasks += build.preassemble
flavours = build.gradle
if flavours == ['yes']:
flavours = []
flavours_cmd = ''.join([capitalize_intact(f) for f in flavours])
gradletasks += ['assemble' + flavours_cmd + 'Release']
adapt_gradle(build_dir)
for name, number, libpath in srclibpaths:
adapt_gradle(libpath)
cmd = [config['gradle']]
if build.gradleprops:
cmd += ['-P' + kv for kv in build.gradleprops]
cmd += ['clean']
p = FDroidPopen(cmd, cwd=root_dir)
elif bmethod == 'kivy':
pass
elif bmethod == 'ant':
logging.info("Cleaning Ant project...")
p = FDroidPopen(['ant', 'clean'], cwd=root_dir)
if p is not None and p.returncode != 0:
raise BuildException("Error cleaning %s:%s" %
(app.id, build.version), p.output)
for root, dirs, files in os.walk(build_dir):
def del_dirs(dl):
for d in dl:
if d in dirs:
shutil.rmtree(os.path.join(root, d))
def del_files(fl):
for f in fl:
if f in files:
os.remove(os.path.join(root, f))
if 'build.gradle' in files:
# Even when running clean, gradle stores task/artifact caches in
# .gradle/ as binary files. To avoid overcomplicating the scanner,
# manually delete them, just like `gradle clean` should have removed
# the build/ dirs.
del_dirs(['build', '.gradle'])
del_files(['gradlew', 'gradlew.bat'])
if 'pom.xml' in files:
del_dirs(['target'])
if any(f in files for f in ['ant.properties', 'project.properties', 'build.xml']):
del_dirs(['bin', 'gen'])
if 'jni' in dirs:
del_dirs(['obj'])
if options.skipscan:
if build.scandelete:
raise BuildException("Refusing to skip source scan since scandelete is present")
else:
# Scan before building...
logging.info("Scanning source for common problems...")
count = scanner.scan_source(build_dir, root_dir, build)
if count > 0:
if force:
logging.warn('Scanner found %d problems' % count)
else:
raise BuildException("Can't build due to %d errors while scanning" % count)
if not options.notarball:
# Build the source tarball right before we build the release...
logging.info("Creating source tarball...")
tarname = common.getsrcname(app, build)
tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz")
def tarexc(f):
return any(f.endswith(s) for s in ['.svn', '.git', '.hg', '.bzr'])
tarball.add(build_dir, tarname, exclude=tarexc)
tarball.close()
# Run a build command if one is required...
if build.build:
logging.info("Running 'build' commands in %s" % root_dir)
cmd = common.replace_config_vars(build.build, build)
# Substitute source library paths into commands...
for name, number, libpath in srclibpaths:
libpath = os.path.relpath(libpath, root_dir)
cmd = cmd.replace('$$' + name + '$$', libpath)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
raise BuildException("Error running build command for %s:%s" %
(app.id, build.version), p.output)
# Build native stuff if required...
if build.buildjni and build.buildjni != ['no']:
logging.info("Building the native code")
jni_components = build.buildjni
if jni_components == ['yes']:
jni_components = ['']
cmd = [os.path.join(ndk_path, "ndk-build"), "-j1"]
for d in jni_components:
if d:
logging.info("Building native code in '%s'" % d)
else:
logging.info("Building native code in the main project")
manifest = os.path.join(root_dir, d, 'AndroidManifest.xml')
if os.path.exists(manifest):
# Read and write the whole AM.xml to fix newlines and avoid
# the ndk r8c or later 'wordlist' errors. The outcome of this
# under gnu/linux is the same as when using tools like
# dos2unix, but the native python way is faster and will
# work in non-unix systems.
manifest_text = open(manifest, 'U').read()
open(manifest, 'w').write(manifest_text)
# In case the AM.xml read was big, free the memory
del manifest_text
p = FDroidPopen(cmd, cwd=os.path.join(root_dir, d))
if p.returncode != 0:
raise BuildException("NDK build failed for %s:%s" % (app.id, build.version), p.output)
p = None
# Build the release...
if bmethod == 'maven':
logging.info("Building Maven project...")
if '@' in build.maven:
maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1])
else:
maven_dir = root_dir
mvncmd = [config['mvn3'], '-Dandroid.sdk.path=' + config['sdk_path'],
'-Dmaven.jar.sign.skip=true', '-Dmaven.test.skip=true',
'-Dandroid.sign.debug=false', '-Dandroid.release=true',
'package']
if build.target:
target = build.target.split('-')[1]
common.regsub_file(r'<platform>[0-9]*</platform>',
r'<platform>%s</platform>' % target,
os.path.join(root_dir, 'pom.xml'))
if '@' in build.maven:
common.regsub_file(r'<platform>[0-9]*</platform>',
r'<platform>%s</platform>' % target,
os.path.join(maven_dir, 'pom.xml'))
p = FDroidPopen(mvncmd, cwd=maven_dir)
bindir = os.path.join(root_dir, 'target')
elif bmethod == 'kivy':
logging.info("Building Kivy project...")
spec = os.path.join(root_dir, 'buildozer.spec')
if not os.path.exists(spec):
raise BuildException("Expected to find buildozer-compatible spec at {0}"
.format(spec))
defaults = {'orientation': 'landscape', 'icon': '',
'permissions': '', 'android.api': "18"}
bconfig = ConfigParser(defaults, allow_no_value=True)
bconfig.read(spec)
distdir = os.path.join('python-for-android', 'dist', 'fdroid')
if os.path.exists(distdir):
shutil.rmtree(distdir)
modules = bconfig.get('app', 'requirements').split(',')
cmd = 'ANDROIDSDK=' + config['sdk_path']
cmd += ' ANDROIDNDK=' + ndk_path
cmd += ' ANDROIDNDKVER=' + build.ndk
cmd += ' ANDROIDAPI=' + str(bconfig.get('app', 'android.api'))
cmd += ' VIRTUALENV=virtualenv'
cmd += ' ./distribute.sh'
cmd += ' -m ' + "'" + ' '.join(modules) + "'"
cmd += ' -d fdroid'
p = subprocess.Popen(cmd, cwd='python-for-android', shell=True)
if p.returncode != 0:
raise BuildException("Distribute build failed")
cid = bconfig.get('app', 'package.domain') + '.' + bconfig.get('app', 'package.name')
if cid != app.id:
raise BuildException("Package ID mismatch between metadata and spec")
orientation = bconfig.get('app', 'orientation', 'landscape')
if orientation == 'all':
orientation = 'sensor'
cmd = ['./build.py'
'--dir', root_dir,
'--name', bconfig.get('app', 'title'),
'--package', app.id,
'--version', bconfig.get('app', 'version'),
'--orientation', orientation
]
perms = bconfig.get('app', 'permissions')
for perm in perms.split(','):
cmd.extend(['--permission', perm])
if config.get('app', 'fullscreen') == 0:
cmd.append('--window')
icon = bconfig.get('app', 'icon.filename')
if icon:
cmd.extend(['--icon', os.path.join(root_dir, icon)])
cmd.append('release')
p = FDroidPopen(cmd, cwd=distdir)
elif bmethod == 'gradle':
logging.info("Building Gradle project...")
cmd = [config['gradle']]
if build.gradleprops:
cmd += ['-P' + kv for kv in build.gradleprops]
cmd += gradletasks
p = FDroidPopen(cmd, cwd=root_dir)
elif bmethod == 'ant':
logging.info("Building Ant project...")
cmd = ['ant']
if build.antcommands:
cmd += build.antcommands
else:
cmd += ['release']
p = FDroidPopen(cmd, cwd=root_dir)
bindir = os.path.join(root_dir, 'bin')
if p is not None and p.returncode != 0:
raise BuildException("Build failed for %s:%s" % (app.id, build.version), p.output)
logging.info("Successfully built version " + build.version + ' of ' + app.id)
omethod = build.output_method()
if omethod == 'maven':
stdout_apk = '\n'.join([
line for line in p.output.splitlines() if any(
a in line for a in ('.apk', '.ap_', '.jar'))])
m = re.match(r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk",
stdout_apk, re.S | re.M)
if not m:
m = re.match(r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]",
stdout_apk, re.S | re.M)
if not m:
m = re.match(r'.*^\[INFO\] [^$]*aapt \[package,[^$]*' + bindir + r'/([^/]+)\.ap[_k][,\]]',
stdout_apk, re.S | re.M)
if not m:
m = re.match(r".*^\[INFO\] Building jar: .*/" + bindir + r"/(.+)\.jar",
stdout_apk, re.S | re.M)
if not m:
raise BuildException('Failed to find output')
src = m.group(1)
src = os.path.join(bindir, src) + '.apk'
elif omethod == 'kivy':
src = os.path.join('python-for-android', 'dist', 'default', 'bin',
'{0}-{1}-release.apk'.format(
bconfig.get('app', 'title'),
bconfig.get('app', 'version')))
elif omethod == 'gradle':
src = None
for apks_dir in [
os.path.join(root_dir, 'build', 'outputs', 'apk'),
os.path.join(root_dir, 'build', 'apk'),
]:
for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']:
apks = glob.glob(os.path.join(apks_dir, apkglob))
if len(apks) > 1:
raise BuildException('More than one resulting apks found in %s' % apks_dir,
'\n'.join(apks))
if len(apks) == 1:
src = apks[0]
break
if src is not None:
break
if src is None:
raise BuildException('Failed to find any output apks')
elif omethod == 'ant':
stdout_apk = '\n'.join([
line for line in p.output.splitlines() if '.apk' in line])
src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk,
re.S | re.M).group(1)
src = os.path.join(bindir, src)
elif omethod == 'raw':
globpath = os.path.join(root_dir, build.output)
apks = glob.glob(globpath)
if len(apks) > 1:
raise BuildException('Multiple apks match %s' % globpath, '\n'.join(apks))
if len(apks) < 1:
raise BuildException('No apks match %s' % globpath)
src = os.path.normpath(apks[0])
# Make sure it's not debuggable...
if common.isApkDebuggable(src, config):
raise BuildException("APK is debuggable")
# By way of a sanity check, make sure the version and version
# code in our new apk match what we expect...
logging.debug("Checking " + src)
if not os.path.exists(src):
raise BuildException("Unsigned apk is not at expected location of " + src)
p = SdkToolsPopen(['aapt', 'dump', 'badging', src], output=False)
vercode = None
version = None
foundid = None
nativecode = None
for line in p.output.splitlines():
if line.startswith("package:"):
pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
m = pat.match(line)
if m:
foundid = m.group(1)
pat = re.compile(".*versionCode='([0-9]*)'.*")
m = pat.match(line)
if m:
vercode = m.group(1)
pat = re.compile(".*versionName='([^']*)'.*")
m = pat.match(line)
if m:
version = m.group(1)
elif line.startswith("native-code:"):
nativecode = line[12:]
# Ignore empty strings or any kind of space/newline chars that we don't
# care about
if nativecode is not None:
nativecode = nativecode.strip()
nativecode = None if not nativecode else nativecode
if build.buildjni and build.buildjni != ['no']:
if nativecode is None:
raise BuildException("Native code should have been built but none was packaged")
if build.novcheck:
vercode = build.vercode
version = build.version
if not version or not vercode:
raise BuildException("Could not find version information in build in output")
if not foundid:
raise BuildException("Could not find package ID in output")
if foundid != app.id:
raise BuildException("Wrong package ID - build " + foundid + " but expected " + app.id)
# Some apps (e.g. Timeriffic) have had the bonkers idea of
# including the entire changelog in the version number. Remove
# it so we can compare. (TODO: might be better to remove it
# before we compile, in fact)
index = version.find(" //")
if index != -1:
version = version[:index]
if (version != build.version or
vercode != build.vercode):
raise BuildException(("Unexpected version/version code in output;"
" APK: '%s' / '%s', "
" Expected: '%s' / '%s'")
% (version, str(vercode), build.version,
str(build.vercode))
)
# Add information for 'fdroid verify' to be able to reproduce the build
# environment.
if onserver:
metadir = os.path.join(tmp_dir, 'META-INF')
if not os.path.exists(metadir):
os.mkdir(metadir)
homedir = os.path.expanduser('~')
for fn in ['buildserverid', 'fdroidserverid']:
shutil.copyfile(os.path.join(homedir, fn),
os.path.join(metadir, fn))
subprocess.call(['jar', 'uf', os.path.abspath(src),
'META-INF/' + fn], cwd=tmp_dir)
# Copy the unsigned apk to our destination directory for further
# processing (by publish.py)...
dest = os.path.join(output_dir, common.getapkname(app, build))
shutil.copyfile(src, dest)
# Move the source tarball into the output directory...
if output_dir != tmp_dir and not options.notarball:
shutil.move(os.path.join(tmp_dir, tarname),
os.path.join(output_dir, tarname))
def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, install, force, onserver):
"""Do a build locally."""
# Prepare the source code...
root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild,
build_dir, srclib_dir, extlib_dir, onserver)
# We need to clean via the build tool in case the binary dirs are
# different from the default ones
p = None
if thisbuild.get('maven', 'no') != 'no':
print "Cleaning Maven project..."
cmd = [config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']]
if '@' in thisbuild['maven']:
maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@',1)[1])
maven_dir = os.path.normpath(maven_dir)
else:
maven_dir = root_dir
p = FDroidPopen(cmd, cwd=maven_dir)
elif thisbuild.get('gradle', 'no') != 'no':
print "Cleaning Gradle project..."
cmd = [config['gradle'], 'clean']
if '@' in thisbuild['gradle']:
gradle_dir = os.path.join(root_dir, thisbuild['gradle'].split('@',1)[1])
gradle_dir = os.path.normpath(gradle_dir)
else:
gradle_dir = root_dir
p = FDroidPopen(cmd, cwd=gradle_dir)
elif thisbuild.get('update', '.') != 'no' and thisbuild.get('kivy', 'no') == 'no':
print "Cleaning Ant project..."
cmd = ['ant', 'clean']
p = FDroidPopen(cmd, cwd=root_dir)
if p is not None and p.returncode != 0:
raise BuildException("Error cleaning %s:%s" %
(app['id'], thisbuild['version']), p.stdout, p.stderr)
# Also clean jni
print "Cleaning jni dirs..."
for baddir in [
'libs/armeabi-v7a', 'libs/armeabi',
'libs/mips', 'libs/x86', 'obj']:
badpath = os.path.join(build_dir, baddir)
if os.path.exists(badpath):
print "Removing '%s'" % badpath
shutil.rmtree(badpath)
# Scan before building...
print "Scanning source for common problems..."
buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
if len(buildprobs) > 0:
print 'Scanner found ' + str(len(buildprobs)) + ' problems:'
for problem in buildprobs:
print '...' + problem
if not force:
raise BuildException("Can't build due to " +
str(len(buildprobs)) + " scanned problems")
# Build the source tarball right before we build the release...
print "Creating source tarball..."
tarname = common.getsrcname(app,thisbuild)
tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz")
def tarexc(f):
for vcs_dir in ['.svn', '.git', '.hg', '.bzr']:
if f.endswith(vcs_dir):
return True
return False
tarball.add(build_dir, tarname, exclude=tarexc)
tarball.close()
# Run a build command if one is required...
if 'build' in thisbuild:
cmd = common.replace_config_vars(thisbuild['build'])
# Substitute source library paths into commands...
for name, number, libpath in srclibpaths:
libpath = os.path.relpath(libpath, root_dir)
cmd = cmd.replace('$$' + name + '$$', libpath)
if options.verbose:
print "Running 'build' commands in %s" % root_dir
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
raise BuildException("Error running build command for %s:%s" %
(app['id'], thisbuild['version']), p.stdout, p.stderr)
# Build native stuff if required...
if thisbuild.get('buildjni') not in (None, 'no'):
print "Building native libraries..."
jni_components = thisbuild.get('buildjni')
if jni_components == 'yes':
jni_components = ['']
else:
jni_components = [c.strip() for c in jni_components.split(';')]
ndkbuild = os.path.join(config['ndk_path'], "ndk-build")
for d in jni_components:
if options.verbose:
print "Running ndk-build in " + root_dir + '/' + d
manifest = root_dir + '/' + d + '/AndroidManifest.xml'
if os.path.exists(manifest):
# Read and write the whole AM.xml to fix newlines and avoid
# the ndk r8c or later 'wordlist' errors. The outcome of this
# under gnu/linux is the same as when using tools like
# dos2unix, but the native python way is faster and will
# work in non-unix systems.
manifest_text = open(manifest, 'U').read()
open(manifest, 'w').write(manifest_text)
# In case the AM.xml read was big, free the memory
del manifest_text
p = FDroidPopen([ndkbuild], cwd=os.path.join(root_dir,d))
if p.returncode != 0:
raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout, p.stderr)
p = None
# Build the release...
if thisbuild.get('maven', 'no') != 'no':
print "Building Maven project..."
if '@' in thisbuild['maven']:
maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@',1)[1])
else:
maven_dir = root_dir
mvncmd = [config['mvn3'], '-Dandroid.sdk.path=' + config['sdk_path']]
if install:
mvncmd += ['-Dandroid.sign.debug=true', 'package', 'android:deploy']
else:
mvncmd += ['-Dandroid.sign.debug=false', '-Dandroid.release=true', 'package']
if 'target' in thisbuild:
target = thisbuild["target"].split('-')[1]
subprocess.call(['sed', '-i',
's@<platform>[0-9]*</platform>@<platform>'+target+'</platform>@g',
'pom.xml'], cwd=root_dir)
if '@' in thisbuild['maven']:
subprocess.call(['sed', '-i',
's@<platform>[0-9]*</platform>@<platform>'+target+'</platform>@g',
'pom.xml'], cwd=maven_dir)
if 'mvnflags' in thisbuild:
mvncmd += thisbuild['mvnflags']
p = FDroidPopen(mvncmd, cwd=maven_dir)
bindir = os.path.join(root_dir, 'target')
elif thisbuild.get('kivy', 'no') != 'no':
print "Building Kivy project..."
spec = os.path.join(root_dir, 'buildozer.spec')
if not os.path.exists(spec):
raise BuildException("Expected to find buildozer-compatible spec at {0}"
.format(spec))
defaults = {'orientation': 'landscape', 'icon': '',
'permissions': '', 'android.api': "18"}
bconfig = ConfigParser(defaults, allow_no_value=True)
bconfig.read(spec)
distdir = 'python-for-android/dist/fdroid'
if os.path.exists(distdir):
shutil.rmtree(distdir)
modules = bconfig.get('app', 'requirements').split(',')
cmd = 'ANDROIDSDK=' + config['sdk_path']
cmd += ' ANDROIDNDK=' + config['ndk_path']
cmd += ' ANDROIDNDKVER=r9'
cmd += ' ANDROIDAPI=' + str(bconfig.get('app', 'android.api'))
cmd += ' VIRTUALENV=virtualenv'
cmd += ' ./distribute.sh'
cmd += ' -m ' + "'" + ' '.join(modules) + "'"
cmd += ' -d fdroid'
if subprocess.call(cmd, cwd='python-for-android', shell=True) != 0:
raise BuildException("Distribute build failed")
cid = bconfig.get('app', 'package.domain') + '.' + bconfig.get('app', 'package.name')
if cid != app['id']:
raise BuildException("Package ID mismatch between metadata and spec")
orientation = bconfig.get('app', 'orientation', 'landscape')
if orientation == 'all':
orientation = 'sensor'
cmd = ['./build.py'
'--dir', root_dir,
'--name', bconfig.get('app', 'title'),
'--package', app['id'],
'--version', bconfig.get('app', 'version'),
'--orientation', orientation,
]
perms = bconfig.get('app', 'permissions')
for perm in perms.split(','):
cmd.extend(['--permission', perm])
if config.get('app', 'fullscreen') == 0:
cmd.append('--window')
icon = bconfig.get('app', 'icon.filename')
if icon:
cmd.extend(['--icon', os.path.join(root_dir, icon)])
cmd.append('release')
p = FDroidPopen(cmd, cwd=distdir)
elif thisbuild.get('gradle', 'no') != 'no':
print "Building Gradle project..."
if '@' in thisbuild['gradle']:
flavour = thisbuild['gradle'].split('@')[0]
gradle_dir = thisbuild['gradle'].split('@')[1]
gradle_dir = os.path.join(root_dir, gradle_dir)
else:
flavour = thisbuild['gradle']
gradle_dir = root_dir
if 'compilesdk' in thisbuild:
level = thisbuild["compilesdk"].split('-')[1]
subprocess.call(['sed', '-i',
's@compileSdkVersion[ ]*[0-9]*@compileSdkVersion '+level+'@g',
'build.gradle'], cwd=root_dir)
if '@' in thisbuild['gradle']:
subprocess.call(['sed', '-i',
's@compileSdkVersion[ ]*[0-9]*@compileSdkVersion '+level+'@g',
'build.gradle'], cwd=gradle_dir)
for root, dirs, files in os.walk(build_dir):
for f in files:
if f == 'build.gradle':
adapt_gradle(os.path.join(root, f))
break
if flavour in ['main', 'yes', '']:
flavour = ''
commands = [config['gradle']]
if 'preassemble' in thisbuild:
for task in thisbuild['preassemble'].split():
commands.append(task)
if install:
commands += ['assemble'+flavour+'Debug', 'install'+flavour+'Debug']
else:
commands += ['assemble'+flavour+'Release']
p = FDroidPopen(commands, cwd=gradle_dir)
else:
print "Building Ant project..."
cmd = ['ant']
if install:
cmd += ['debug','install']
elif 'antcommand' in thisbuild:
cmd += [thisbuild['antcommand']]
else:
cmd += ['release']
p = FDroidPopen(cmd, cwd=root_dir)
bindir = os.path.join(root_dir, 'bin')
if p.returncode != 0:
raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout, p.stderr)
print "Successfully built version " + thisbuild['version'] + ' of ' + app['id']
if install:
return
# Find the apk name in the output...
if 'bindir' in thisbuild:
bindir = os.path.join(build_dir, thisbuild['bindir'])
if thisbuild.get('maven', 'no') != 'no':
stdout_apk = '\n'.join([
line for line in p.stdout.splitlines() if any(a in line for a in ('.apk','.ap_'))])
m = re.match(r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk",
stdout_apk, re.S|re.M)
if not m:
m = re.match(r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]",
stdout_apk, re.S|re.M)
if not m:
m = re.match(r'.*^\[INFO\] [^$]*aapt \[package,[^$]*' + bindir + r'/([^/]+)\.ap[_k][,\]]',
stdout_apk, re.S|re.M)
if not m:
raise BuildException('Failed to find output')
src = m.group(1)
src = os.path.join(bindir, src) + '.apk'
elif thisbuild.get('kivy', 'no') != 'no':
src = 'python-for-android/dist/default/bin/{0}-{1}-release.apk'.format(
bconfig.get('app', 'title'), bconfig.get('app', 'version'))
elif thisbuild.get('gradle', 'no') != 'no':
dd = build_dir
if 'subdir' in thisbuild:
dd = os.path.join(dd, thisbuild['subdir'])
if flavour in ['main', 'yes', '']:
name = '-'.join([os.path.basename(dd), 'release', 'unsigned'])
else:
name = '-'.join([os.path.basename(dd), flavour, 'release', 'unsigned'])
src = os.path.join(dd, 'build', 'apk', name+'.apk')
else:
stdout_apk = '\n'.join([
line for line in p.stdout.splitlines() if '.apk' in line])
src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk,
re.S|re.M).group(1)
src = os.path.join(bindir, src)
# Make sure it's not debuggable...
if common.isApkDebuggable(src, config):
raise BuildException("APK is debuggable")
# By way of a sanity check, make sure the version and version
# code in our new apk match what we expect...
print "Checking " + src
if not os.path.exists(src):
raise BuildException("Unsigned apk is not at expected location of " + src)
p = subprocess.Popen([os.path.join(config['sdk_path'],
'build-tools', config['build_tools'], 'aapt'),
'dump', 'badging', src],
stdout=subprocess.PIPE)
output = p.communicate()[0]
vercode = None
version = None
foundid = None
for line in output.splitlines():
if line.startswith("package:"):
pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
foundid = re.match(pat, line).group(1)
pat = re.compile(".*versionCode='([0-9]*)'.*")
vercode = re.match(pat, line).group(1)
pat = re.compile(".*versionName='([^']*)'.*")
version = re.match(pat, line).group(1)
if thisbuild['novcheck']:
vercode = thisbuild['vercode']
version = thisbuild['version']
if not version or not vercode:
raise BuildException("Could not find version information in build in output")
if not foundid:
raise BuildException("Could not find package ID in output")
if foundid != app['id']:
raise BuildException("Wrong package ID - build " + foundid + " but expected " + app['id'])
# Some apps (e.g. Timeriffic) have had the bonkers idea of
# including the entire changelog in the version number. Remove
# it so we can compare. (TODO: might be better to remove it
# before we compile, in fact)
index = version.find(" //")
if index != -1:
version = version[:index]
if (version != thisbuild['version'] or
vercode != thisbuild['vercode']):
raise BuildException(("Unexpected version/version code in output;"
" APK: '%s' / '%s', "
" Expected: '%s' / '%s'")
% (version, str(vercode), thisbuild['version'], str(thisbuild['vercode']))
)
# Copy the unsigned apk to our destination directory for further
# processing (by publish.py)...
dest = os.path.join(output_dir, common.getapkname(app,thisbuild))
shutil.copyfile(src, dest)
# Move the source tarball into the output directory...
if output_dir != tmp_dir:
shutil.move(os.path.join(tmp_dir, tarname),
os.path.join(output_dir, tarname))
perm = re.match(string_pat, line).group(1)
# Filter out this, it's only added with the latest SDK tools and
# causes problems for lots of apps.
if perm != "android.hardware.screen.portrait" \
and perm != "android.hardware.screen.landscape":
if perm.startswith("android.feature."):
perm = perm[16:]
thisinfo['features'].add(perm)
if 'sdkversion' not in thisinfo:
logging.warn(
"No SDK version information found in {0}".format(apkfile))
thisinfo['sdkversion'] = 0
# Check for debuggable apks...
if common.isApkDebuggable(apkfile, config):
logging.warn(
'{0} is set to android:debuggable="true"'.format(apkfile))
# Calculate the sha256...
sha = hashlib.sha256()
with open(apkfile, 'rb') as f:
while True:
t = f.read(1024)
if len(t) == 0:
break
sha.update(t)
thisinfo['sha256'] = sha.hexdigest()
# Get the signature (or md5 of, to be precise)...
getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
